Misc geodata provisioning code

This commit is contained in:
Rick Falkvinge
2014-12-13 22:22:36 +01:00
parent 35ed03695e
commit ce0d77ced2
7 changed files with 86 additions and 30 deletions
+12 -2
View File
@@ -117,6 +117,9 @@
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="CentralEurope" xml:space="preserve">
<value>Central Europe</value>
</data>
<data name="Country_DK" xml:space="preserve">
<value>Denmark</value>
</data>
@@ -126,6 +129,9 @@
<data name="Country_GH" xml:space="preserve">
<value>Ghana</value>
</data>
<data name="Country_NL" xml:space="preserve">
<value>Netherlands</value>
</data>
<data name="Country_NO" xml:space="preserve">
<value>Norway</value>
</data>
@@ -134,7 +140,6 @@
</data>
<data name="EasternEurope" xml:space="preserve">
<value>Eastern Europe</value>
<comment>(East of southern tip of Sweden, appx)</comment>
</data>
<data name="Europe" xml:space="preserve">
<value>Europe</value>
@@ -147,6 +152,12 @@
<value>Nordics</value>
<comment>(Finland Denmark Norway Sweden Iceland)</comment>
</data>
<data name="NorthernEurope" xml:space="preserve">
<value>Nothern Europe</value>
</data>
<data name="SouthernEurope" xml:space="preserve">
<value>Southern Europe</value>
</data>
<data name="USA_EastNorthCentral" xml:space="preserve">
<value>East North Central</value>
</data>
@@ -188,7 +199,6 @@
</data>
<data name="WesternEurope" xml:space="preserve">
<value>Western Europe</value>
<comment>(West of southern tip of Sweden, appx)</comment>
</data>
<data name="World" xml:space="preserve">
<value>World</value>
@@ -89,7 +89,7 @@
</div>
<div class="entryLabels">
Upload file for processing<br/><br/>
Upload <strong>geodata</strong> for processing<br/><br/>
</div>
</div>
<div clear="both"></div>
@@ -40,15 +40,15 @@ namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
//
// Tab separated fields
//
// [countrycode] [tab] [geo1] .. [geo_n]
// [countrycode] [tab] [geo1] [tab] [geo2] [tab] .. [tab] [geo_n]
//
// example:
//
// NL [Tab] Greater Amsterdam
// NL [Tab] Greater Amsterdam [Tab] Amsterdam
// NL [Tab] Greater Amsterdam [Tab] Amsterdam [Tab] Wallen
// NL [Tab] Noord-Holland [Tab] Greater Amsterdam
// NL [Tab] Noord-Holland [Tab] Greater Amsterdam [Tab] Amsterdam
// NL [Tab] Noord-Holland [Tab] Greater Amsterdam [Tab] Amsterdam [Tab] Wallen
//
// To populate all four nodes, only the last line is necessary, but all three lines are valid.
// To populate all five nodes (including country), only the last line is necessary, but all these lines are valid.
//
// Nodes are not re-populated on repetition. Node names within a country scope are required to
// be unique.
@@ -89,7 +89,7 @@
</div>
<div class="entryLabels">
Upload file for processing<br/><br/>
Upload <strong>postal codes</strong> for processing<br/><br/>
</div>
</div>
<div clear="both"></div>
@@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Threading;
using System.Web.Services;
using Swarmops.Basic.Enums;
using Swarmops.Database;
using Swarmops.Logic.Cache;
using Swarmops.Logic.Security;
using Swarmops.Logic.Structure;
@@ -17,7 +19,7 @@ namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
{
protected void Page_Load (object sender, EventArgs e)
{
PageTitle = "Populate Country - Postal Codes, Cities";
PageTitle = "Populate Country 2 - Postal Codes, Cities";
PageIcon = "iconshock-battery-drill";
InfoBoxLiteral = "You've taken a wrong turn if you're even seeing this page. It's for development purposes.";
}
@@ -33,6 +35,22 @@ namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
{
// Start an async thread that does all the work, then return
// FORMAT OF FILE
//
// Tab separated fields
//
// [countrycode] [tab] [postalCode] [tab] [cityName] [tab] [geoNodeName]
//
// example:
//
// NL [Tab] 1026 [Tab] Amsterdam-Zuid [Tab] Amsterdam
//
// For countries that don't use postal codes, leave the postal code field empty.
//
// Node names within a country scope are required to be unique.
if (!PilotInstallationIds.IsPilot(PilotInstallationIds.SwarmopsLive) && !Debugger.IsAttached)
{
throw new UnauthorizedAccessException("This may only run on Swarmops Master");
@@ -70,38 +88,65 @@ namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
Random random = new Random();
Dictionary<string,bool> postalCodeDupes = new Dictionary<string, bool>();
foreach (string lineRaw in lines)
{
count++;
string line = lineRaw.Trim();
string[] lineParts = line.Split ('\t');
if (lineParts.Length < 12)
if (lineParts.Length != 4)
{
continue;
}
int percent = (count*99)/lines.Length;
int percent = (count * 99) / lines.Length;
if (percent == 0)
{
percent = 1;
}
GuidCache.Set (guid + "-Progress", percent);
string name = lineParts[0] + " " + lineParts[1];
PersonGender gender = lineParts[2] == "male" ? PersonGender.Male : PersonGender.Female;
DateTime dateOfBirth = DateTime.Parse (lineParts[7], new CultureInfo ("en-US"), DateTimeStyles.None);
Country country = Country.FromCode (lineParts[6]);
GuidCache.Set(guid + "-Progress", percent);
Person newPerson = Person.Create (name, string.Empty, string.Empty, lineParts[8], lineParts[3],
lineParts[4].Replace (" ", ""), lineParts[5], lineParts[6], dateOfBirth, gender);
string countryCode = lineParts[0].Trim().ToUpperInvariant();
string postalCode = lineParts[1].Trim();
string cityName = lineParts[2].Trim();
string nodeName = lineParts[3].Trim();
newPerson.NationalIdNumber = lineParts[9];
newPerson.Longitude = lineParts[10];
newPerson.Latitude = lineParts[11];
// Dupecheck
newPerson.AddMembership (1, DateTime.Today.AddDays (random.Next (365)));
if (postalCodeDupes.ContainsKey (postalCode))
{
continue;
}
Geography geography = Geography.FromName (nodeName); // may dupe!
// First, make sure country exists
Country country = Country.FromCode (countryCode);
// Then, check if the city name exists
City city = null;
try
{
city = City.FromName (cityName, country.Identity);
}
catch (ArgumentException)
{
city = City.Create (cityName, country.Identity, geography.Identity);
}
// Last, add the postal code, if there is any
if (!string.IsNullOrEmpty (postalCode))
{
SwarmDb.GetDatabaseForWriting().CreatePostalCode (postalCode, city.Identity, country.Identity);
postalCodeDupes[postalCode] = true;
}
}
GuidCache.Set (guid + "-Progress", 100);
@@ -1,4 +1,4 @@
<%@ Page Title="" Language="C#" MasterPageFile="~/Master-v5.master" AutoEventWireup="true" CodeFile="PopulateData.aspx.cs" Inherits="Swarmops.Frontend.Pages.v5.Admin.Hacks.PopulateData" %>
<%@ Page Title="" Language="C#" MasterPageFile="~/Master-v5.master" AutoEventWireup="true" CodeFile="PopulateFakePeople.aspx.cs" Inherits="Swarmops.Frontend.Pages.v5.Admin.Hacks.PopulateFakePeople" %>
<%@ Register src="~/Controls/v5/Base/FileUpload.ascx" tagname="FileUpload" tagprefix="Swarmops5" %>
<%@ Register TagPrefix="Swarmops5" TagName="ExternalScripts" Src="~/Controls/v5/UI/ExternalScripts.ascx" %>
<%@ Register TagPrefix="Swarmops5" TagName="ComboBudgets" Src="~/Controls/v5/Financial/ComboBudgets.ascx" %>
@@ -21,7 +21,7 @@
$.ajax({
type: "POST",
url: "PopulateData.aspx/InitializeProcessing",
url: "PopulateFakePeople.aspx/InitializeProcessing",
data: "{'guid': '<%= this.UploadFile.GuidString %>'}",
contentType: "application/json; charset=utf-8",
dataType: "json",
@@ -89,7 +89,7 @@
</div>
<div class="entryLabels">
Upload file for processing<br/><br/>
Upload <strong>fake people</strong> for processing<br/><br/>
</div>
</div>
<div clear="both"></div>
@@ -1,6 +1,7 @@
using System;
using System.Globalization;
using System.IO;
using System.Text;
using System.Threading;
using System.Web.Services;
using Swarmops.Basic.Enums;
@@ -12,11 +13,11 @@ using Swarmops.Logic.Swarm;
namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
{
public partial class PopulateData : PageV5Base
public partial class PopulateFakePeople : PageV5Base
{
protected void Page_Load (object sender, EventArgs e)
{
PageTitle = "Populate Data";
PageTitle = "Populate Data - Fake People";
PageIcon = "iconshock-battery-drill";
InfoBoxLiteral = "You've taken a wrong turn if you're even seeing this page. It's for development purposes.";
}
@@ -53,7 +54,7 @@ namespace Swarmops.Frontend.Pages.v5.Admin.Hacks
string data = string.Empty;
int count = 0;
using (StreamReader reader = uploadedDoc.GetReader (1252))
using (StreamReader reader = uploadedDoc.GetReader (Encoding.UTF8))
{
data = reader.ReadToEnd();
}