<?php
/*
cat mexico.geojson | wc
sed ':a;N;$!ba;s/\n/ /g' mexico.geojson > mexico.geojson_out1
cat mexico.geojson_out1 | wc
http://localhost/Utility/cord/largefile.php?filename=mexico.geojson_out1
sed -E 's/\{ "type": "FeatureCollection", "name": "mexico_administrative_osm_province_boundaries", "crs": \{ "type": "name", "properties": \{ "name": "urn:ogc:def:crs:OGC:1.3:CRS84" \} \}, "features": \[//g' ./mexico.geojson_out1 > mexico.geojson_out2
http://localhost/Utility/cord/largefile.php?filename=mexico.geojson_out2
sed -E 's/(\[|\{|\}|\])/ \1 /g' ./mexico.geojson_out2 > ./output/mexico.geojson_output
http://localhost/Utility/cord/largefile.php?filename=output/mexico.geojson_output
*/
ini_set('max_execution_time', '500'); //300 seconds = 5 minutes
ini_set('memory_limit', '1G');
//ini_set('max_execution_time', '0'); // for infinite time of execution
define('CHUNK_SIZE', 780*102); // Size (in bytes) of tiles chunk
//define('CHUNK_SIZE', 10*102); // Size (in bytes) of tiles chunk
global $counter;
global $conn;
global $fieldnames;
$counter = 0;
$servername = "localhost";
$username = "shanky";
$password = "Shanky@123";
$dbname = "geo";
// Create connection
$conn = new mysqli($servername, $username, $password, $dbname);
// Check connection
if ($conn->connect_error) {
die("Connection failed: " . $conn->connect_error);
}
function insertToDb($record,$table,$count){
global $conn;
global $fieldnames;
$data = json_decode($record);
if($count == 1)
{
$fields[] = '`uid` int(11) NOT NULL';
foreach($data->properties as $key => $value)
{
$fieldnames[] = $key;
$fields[] = '`'.$key.'` varchar(50) NULL';
}
$fields[] = '`coordinates` longtext NOT NULL';
$fieldnames[] = 'coordinates';
$q = "cREATE TABLE `".$table."` (".implode(",",$fields).") ENGINE=InnoDB DEFAULT CHARSET=latin1";
$conn->query($q);
$q = "aLTER TABLE `".$table."` ADD PRIMARY KEY (`uid`)";
$conn->query($q);
$q = "aLTER TABLE `".$table."` MODIFY `uid` int(11) NOT NULL AUTO_INCREMENT;";
$conn->query($q);
}
foreach ($fieldnames as $key => $value) {
if($value == "coordinates")
{
$fieldValues[] = "'".$conn->real_escape_string(json_encode($data->geometry->coordinates[0][0]))."'";
}
else
{
$fieldValues[] = "'".$conn->real_escape_string($data->properties->{$value})."'";
}
}
//$sql = "INSERT INTO ".$table." (".implode(",",$fieldnames).") VALUES ('".$conn->real_escape_string($data->properties->gid)."', '".$conn->real_escape_string($data->properties->state)."', '".$conn->real_escape_string($data->properties->county)."','".$conn->real_escape_string($data->properties->geoid)."','".$conn->real_escape_string(json_encode($data->geometry->coordinates[0][0]))."')";
echo $sql = "INSERT INTO ".$table." (`".implode("`,`",$fieldnames)."`) VALUES (".implode(",",$fieldValues).")";
if ($conn->query($sql) !== TRUE) {
print_r($data);
die();
}
}
function readfile_chunked($filename, $retbytes = TRUE) {
global $counter;
$table = '';
$buffer = '';
$cnt = 0;
$handle = fopen($filename, 'rb');
if ($handle === false) {
return false;
}
$stackOperator = [];
$stackBufferString = [];
$currentStackCount = 0;
$bufferString = "";
$record = [];
while (!feof($handle)) {
//if($counter == 1){ return $record; } echo $counter++;
$buffer = fread($handle, CHUNK_SIZE);
$a = explode(" ", $buffer);
if(isset($_POST['Run']) && $_POST['Run'] != "")
{
$table = $_POST['Run'];
}
else{
if($counter == 2){ return $record; }else{ echo $buffer; } $counter++;
}
for($i=0; $i < count($a); $i++)
{
if($a[$i] == ""){
continue;
}
elseif($a[$i] == "[" || $a[$i] == "{")
{
$stackOperator[] = $a[$i];
$stackBufferString[] = "";
}
elseif(($a[$i] == "]" && $stackOperator[count($stackOperator)-1] == "[") ||
($a[$i] == "}" && $stackOperator[count($stackOperator)-1] == "{") )
{
$op = array_pop($stackOperator);
$buffer = $op.array_pop($stackBufferString).$a[$i];
if(count($stackOperator) == 0){
$record[] = $buffer;
if($table != "")
{
insertToDb($buffer,$table,count($record));
}
}else{
$stackBufferString[count($stackOperator)-1] .= $buffer;
}
}
else{
$stackBufferString[count($stackOperator)-1] .= $a[$i];
}
}
ob_flush();
flush();
if ($retbytes) {
$cnt += strlen($buffer);
}
//echo "<br />".count($record);
}
$status = fclose($handle);
if ($retbytes && $status) {
return $cnt; // return num. bytes delivered like readfile() does.
}
return $record;;
}
$file = isset($_GET['filename'])?__DIR__."/".$_GET['filename']:"";
if($file != ""){
echo '<form method="post"><input type="submit" name="Run" value="'.(explode(".",basename($file))[0]).'" /></form>';
$arr = readfile_chunked($file);
print_r(count($arr));
}else{
echo "No File";
}
$conn->close();
/*foreach($arr as $value)
{
echo "<pre>";
print_r(json_decode($value));
echo "</pre>";
}*/
You need to save it in the Packages folder hierarchy for ST to pick it up - you can try Tools -> Developer - New Syntax, paste in the tmLanguage and save it in the folder ST recommends (Packages/User) as <packagefile>.tmLanguage , then Tools -> Developer -> New Syntax From <packagefile> .tmLanguage.
Comments
Post a Comment