Skip to main content

Reading Large JSON file through PHP Script

 <?php
/*
cat mexico.geojson | wc
sed ':a;N;$!ba;s/\n/ /g' mexico.geojson  > mexico.geojson_out1
cat mexico.geojson_out1 | wc
http://localhost/Utility/cord/largefile.php?filename=mexico.geojson_out1
sed -E 's/\{ "type": "FeatureCollection", "name": "mexico_administrative_osm_province_boundaries", "crs": \{ "type": "name", "properties": \{ "name": "urn:ogc:def:crs:OGC:1.3:CRS84" \} \}, "features": \[//g' ./mexico.geojson_out1  > mexico.geojson_out2
http://localhost/Utility/cord/largefile.php?filename=mexico.geojson_out2
sed -E 's/(\[|\{|\}|\])/ \1 /g' ./mexico.geojson_out2 > ./output/mexico.geojson_output
http://localhost/Utility/cord/largefile.php?filename=output/mexico.geojson_output

*/

ini_set('max_execution_time', '500'); //300 seconds = 5 minutes
ini_set('memory_limit', '1G');  
//ini_set('max_execution_time', '0'); // for infinite time of execution

define('CHUNK_SIZE', 780*102); // Size (in bytes) of tiles chunk
//define('CHUNK_SIZE', 10*102); // Size (in bytes) of tiles chunk
global $counter;
global $conn;
global $fieldnames;
$counter = 0;



$servername = "localhost";
$username = "shanky";
$password = "Shanky@123";
$dbname = "geo";

// Create connection
$conn = new mysqli($servername, $username, $password, $dbname);
// Check connection
if ($conn->connect_error) {
  die("Connection failed: " . $conn->connect_error);
}

function insertToDb($record,$table,$count){
    global $conn;
    global $fieldnames;
    $data = json_decode($record);


    if($count == 1)
    {
        $fields[] = '`uid` int(11) NOT NULL';
        foreach($data->properties as $key => $value)
        {
            $fieldnames[] = $key;
            $fields[] = '`'.$key.'` varchar(50) NULL';
        }
        $fields[] = '`coordinates` longtext NOT NULL';
        $fieldnames[] = 'coordinates';

        $q = "cREATE TABLE `".$table."` (".implode(",",$fields).") ENGINE=InnoDB DEFAULT CHARSET=latin1";
        $conn->query($q);
        $q = "aLTER TABLE `".$table."` ADD PRIMARY KEY (`uid`)";
        $conn->query($q);
        $q = "aLTER TABLE `".$table."` MODIFY `uid` int(11) NOT NULL AUTO_INCREMENT;";
        $conn->query($q);
      
     }

    

    foreach ($fieldnames as $key => $value) {
        
        if($value == "coordinates")
        {
             $fieldValues[] = "'".$conn->real_escape_string(json_encode($data->geometry->coordinates[0][0]))."'";
        }
        else
        {
            $fieldValues[] = "'".$conn->real_escape_string($data->properties->{$value})."'";
        }
    }


    //$sql = "INSERT INTO ".$table." (".implode(",",$fieldnames).") VALUES ('".$conn->real_escape_string($data->properties->gid)."', '".$conn->real_escape_string($data->properties->state)."', '".$conn->real_escape_string($data->properties->county)."','".$conn->real_escape_string($data->properties->geoid)."','".$conn->real_escape_string(json_encode($data->geometry->coordinates[0][0]))."')";
    echo $sql = "INSERT INTO ".$table." (`".implode("`,`",$fieldnames)."`) VALUES (".implode(",",$fieldValues).")";
    
    if ($conn->query($sql) !== TRUE) {
        print_r($data);
        die();      
    }
    

}

function readfile_chunked($filename, $retbytes = TRUE) {
    global $counter;
    $table = '';
    $buffer = '';
    $cnt    = 0;
    $handle = fopen($filename, 'rb');

    if ($handle === false) {
        return false;
    }
    $stackOperator = [];
    $stackBufferString = [];
    $currentStackCount = 0;
    $bufferString = "";
    $record = [];

    

    while (!feof($handle)) {
        
        //if($counter == 1){ return $record; } echo $counter++;

        $buffer = fread($handle, CHUNK_SIZE);
        $a = explode(" ", $buffer);

        if(isset($_POST['Run']) && $_POST['Run'] != "")
        {
            $table = $_POST['Run'];
        }
        else{
            if($counter == 2){ return $record; }else{ echo $buffer; }  $counter++;
        }
        
        for($i=0; $i < count($a); $i++)
        {
            if($a[$i] == ""){
                continue;
            }
            elseif($a[$i] == "[" || $a[$i] == "{")
            {
                $stackOperator[] = $a[$i];                
                $stackBufferString[] = "";                
            }
            elseif(($a[$i] == "]" && $stackOperator[count($stackOperator)-1] == "[") ||
                    ($a[$i] == "}" && $stackOperator[count($stackOperator)-1] == "{") )
            {
                
                $op = array_pop($stackOperator);
                $buffer = $op.array_pop($stackBufferString).$a[$i];
                
                
                if(count($stackOperator) == 0){
                    $record[] = $buffer;
                    if($table != "")
                    {
                        insertToDb($buffer,$table,count($record));
                    }

                }else{
                    $stackBufferString[count($stackOperator)-1] .= $buffer;
                }
            }
            else{
                $stackBufferString[count($stackOperator)-1] .= $a[$i];
            }           
            
        }
       
        ob_flush();
        flush();

        if ($retbytes) {
            $cnt += strlen($buffer);
        }
        //echo "<br />".count($record);        
    }

    $status = fclose($handle);

    if ($retbytes && $status) {
        return $cnt; // return num. bytes delivered like readfile() does.
    }

    return $record;;
}

     $file  = isset($_GET['filename'])?__DIR__."/".$_GET['filename']:"";
    if($file != ""){
        echo '<form method="post"><input type="submit" name="Run" value="'.(explode(".",basename($file))[0]).'" /></form>';
        $arr =  readfile_chunked($file);
        print_r(count($arr));
    }else{
        echo "No File";
    }
    $conn->close();


    /*foreach($arr as $value)
    {
        echo "<pre>";
        print_r(json_decode($value));
        echo "</pre>";
        
    }*/

Comments

Popular posts from this blog

Enable Https by self-signing certification.

  Run make-ssl-cert command > sudo make-ssl-cert generate-default-snakeoil will created 2 files in following directory.     - /etc/ssl/certs/ssl-cert-snakeoil.pem     - /etc/ssl/private/ssl-cert-snakeoil.key Open /etc/apache2/site-avilable and find and modify with below line (with above certs paths) .         SSLCertificateFile    /etc/ssl/certs/ssl-cert-snakeoil.pem         SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key > sudo a2ensite default-ssl > sudo a2enmode ssl > sudo systemctl restart apache2 Open https://localhost

simple recursive function to copy entire directories

<?php function recurse_copy ( $src , $dst ) {     $dir = opendir ( $src );     @ mkdir ( $dst );     while( false !== ( $file = readdir ( $dir )) ) {         if (( $file != '.' ) && ( $file != '..' )) {             if ( is_dir ( $src . '/' . $file ) ) {                 recurse_copy ( $src . '/' . $file , $dst . '/' . $file );             }             else {                 copy ( $src . '/' . $file , $dst . '/' . $file );             }         }     }     closedir ( $dir ); } ?>