first commit

CRONS/common/Api.php

@@ -0,0 +1,117 @@
+<?php
+class Api
+{
+
+    private static $instance;
+    private static $httpAuthToken;
+    private static $encryptionAlg;
+    private static $encryptionKey;
+    private static $encryptionIV;
+    private static $baseURL;
+
+    public function __construct()
+    {
+    }
+
+    public static function initConfig()
+    {
+        global $savvyext;
+        self::$httpAuthToken = $savvyext->cfgReadChar('system.oauth2_token');
+        self::$encryptionAlg = $savvyext->cfgReadChar('encryption.algorithm');
+        self::$encryptionKey = $savvyext->cfgReadChar('encryption.key');
+        self::$encryptionIV = $savvyext->cfgReadChar('encryption.iv');
+        self::$baseURL = $savvyext->cfgReadChar('system.api_url');
+    }
+
+    public static function getInstance()
+    {
+        if (null === static::$instance) {
+            static::$instance = new static;
+            self::initConfig();
+            return static::$instance;
+        }
+        return static::$instance;
+    }
+
+    public static function postData($endpoint, $payload)
+    {
+        self::getInstance();
+        $encrypted_payload = bin2hex(
+            openssl_encrypt(
+                $payload,
+                self::$encryptionAlg,
+                self::$encryptionKey,
+                OPENSSL_RAW_DATA,
+                self::$encryptionIV
+            ));
+        $postdata = "{\"encrypted_payload\": \"${encrypted_payload}\"}";
+        $url = self::$baseURL . $endpoint;
+        $ch = curl_init($url);
+        curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
+        curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
+        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
+        curl_setopt($ch, CURLOPT_VERBOSE, false);
+        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+                'Content-Type: application/json',
+                'Content-Length: ' . strlen($postdata),
+                'Authorization: Server-Token ' . self::$httpAuthToken,
+                "client_id: BATCH"
+            )
+        );
+
+        $body = curl_exec($ch);
+        $result = json_decode($body, true);
+
+        $decrypted = openssl_decrypt(
+            hex2bin(
+                $result['payload']
+            ),
+            self::$encryptionAlg,
+            self::$encryptionKey,
+            OPENSSL_RAW_DATA,
+            self::$encryptionIV
+        );
+        $payload = json_decode($decrypted, true);
+        return $payload;
+    }
+
+    public static function getData($endpoint)
+    {
+        self::getInstance();
+        if ($endpoint != "") { // minimal sanity
+            sleep(1);
+            $url = self::$baseURL . $endpoint;
+            $ch = curl_init($url);
+            curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "GET");
+            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+            curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
+            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
+            curl_setopt($ch, CURLOPT_VERBOSE, false);
+            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+            curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+                    'Content-Type: application/json',
+                    'Authorization: Server-Token ' . self::$httpAuthToken,
+                    "client_id: BATCH"
+                )
+            );
+
+            $body = curl_exec($ch);
+            $result = json_decode($body, true);
+            $decrypted = openssl_decrypt(
+                hex2bin(
+                    $result['payload']
+                ),
+                self::$encryptionAlg,
+                self::$encryptionKey,
+                OPENSSL_RAW_DATA,
+                self::$encryptionIV
+            );
+            $payload = json_decode($decrypted, true);
+            return [$payload, $decrypted, $result, $body];
+        }
+        return [null, null, null, null];
+    }
+}

CRONS/common/Geocode.php

@@ -0,0 +1,121 @@
+<?php
+class Geocode
+{
+    public static function geocodeAddress($address)
+    {
+        global $httpAuthToken, $oauth_api;
+        if (!empty($address)) {
+            $data = http_build_query(
+                array(
+                    'address' => $address,
+                )
+            );
+            $url = $oauth_api . "geocode?" . $data;
+            $opts = array(
+                'http' => array(
+                    'method' => "GET",
+                    'timeout' => 60, /* 1 minute */
+                    'header' =>
+                    "Content-Type: application/x-www-form-urlencoded\r\n" .
+                    "Accept: application/json\r\n" .
+                    "Authorization: Server-Token ${httpAuthToken}\r\n",
+                ),
+            );
+            $context = stream_context_create($opts);
+            $body = file_get_contents($url, false, $context);
+            $geocoded = json_decode($body, true);
+            if (is_array($geocoded) && isset($geocoded["data"]) && !isset($geocoded["error"])) {
+                return $geocoded["data"];
+            }
+        }
+        return null;
+    }
+
+    public static function reverseGPS($latitude, $longitude)
+    {
+        global $httpAuthToken, $oauth_api;
+       
+        $body = null;
+        if (!empty($latitude && !empty($longitude))) {
+            $data = http_build_query(
+                array(
+                    'lat' => $latitude,
+                    'lng' => $longitude,
+                )
+            );
+
+            $url = $oauth_api . "reverse?" . $data;
+            $opts = array(
+                'http' => array(
+                    'method' => "GET",
+                    'timeout' => 60, /* 1 minute */
+                    'header' =>
+                    "Content-Type: application/x-www-form-urlencoded\r\n" .
+                    "Accept: application/json\r\n" .
+                    "Authorization: Server-Token ${httpAuthToken}\r\n",
+                ),
+            );
+            $context = stream_context_create($opts);
+            $body = file_get_contents($url, false, $context);
+            $geocoded = json_decode($body, true);
+
+            if (is_array($geocoded) && isset($geocoded["data"]) && !isset($geocoded["error"])) {
+                return $geocoded["data"];
+
+            }
+        }
+        return null;
+    }
+
+    public function getTimeZone($lat, $lng)
+    {
+        global $googleKey;
+        $result = array("message" => "Unexpected error", "dstOffset" => 0, "rawOffset" => 0, "status" => "Error", "timeZoneId" => "", "timeZoneName" => "");
+        $url = "https://maps.googleapis.com/maps/api/timezone/json?location=${lat},${lng}&timestamp=" . time() . "&key=" . $googleKey;
+        $resp_json = file_get_contents($url);
+        $resp = json_decode($resp_json, true);
+        if ($resp["status"] == 'OK') {
+            $result = $resp;
+            if ($resp["timeZoneId"] != "") {
+                $result["message"] = "";
+            } else {
+                $result["message"] = "Failed to get timezone offset for: ${lat},${lng}\n";
+            }
+        } else {
+            $result["message"] = "Invalid service response code\n";
+        }
+        return $result;
+    }
+
+    public static function getGeoInfoBigDataCloud($latitude, $longitude)
+    {
+        global $city_keywords;
+        $url = "https://api.bigdatacloud.net/data/reverse-geocode-client?latitude=" . $latitude . "&longitude=" . $longitude . "&localityLanguage=en";
+        $opts = array(
+            'http' => array(
+                'method' => "GET",
+            ),
+        );
+
+        $context = stream_context_create($opts);
+        $body = file_get_contents($url, false, $context);
+        $geoinfo = json_decode($body, true);
+        if (count($geoinfo) > 0 && isset($geoinfo['localityInfo'])) {
+            $localities = $geoinfo['localityInfo']['administrative'];
+            foreach ($localities as $k => $v) {
+                $name = isset($v['name']) ? $v['name'] : "";
+                $description = isset($v['description']) ? $v['description'] : "";
+                $info = strtolower($name . $description);
+                foreach ($city_keywords as $key) {
+                    if (strpos($info, $key) !== false) {
+                        return $v['name'];
+                    }
+                }
+            }
+
+        }
+        return null;
+
+    }
+
+}

CRONS/common/Logger.php

@@ -0,0 +1,103 @@
+<?php
+require __DIR__ . '../../../vendor/autoload.php';
+class Logger
+{
+    protected static $instance;
+    private static $logger;
+
+    const ERROR    = 0;
+    const WARNING  = 1;
+    const INFO     = 2;
+    const DEBUG    = 3;
+    const DEBUG1   = 4;
+    const DEBUG2   = 5;
+    const DEBUG3   = 6;
+    const DEBUG4   = 7;
+    const SQL      = 8;
+    const FLOG_MAX = 9;
+
+    const LEVELS = [
+        'ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEBUG1', 'DEBUG2', 'DEBUG3', 'DEBUG4', 'SQL', 'FLOG_MAX'
+    ];
+
+    public function __construct()
+    {
+    }
+
+    public static function getLogger()
+    {
+        if (!self::$instance) {
+            self::initLogger();
+        }
+
+        return self::$instance;
+    }
+
+    public static function initLogger()
+    {
+        global $savvyext;
+        $fluent_host = $savvyext->cfgReadChar('phplogger.host');
+        $fluent_port = $savvyext->cfgReadLong('phplogger.port');
+        $log_enabled = $savvyext->cfgReadLong('phplogger.enabled');
+
+        if ($log_enabled == 1) {
+            self::$logger = new Fluent\Logger\FluentLogger($fluent_host, $fluent_port);
+        }
+        self::$instance = self::$logger;
+    }
+
+    public static function __callStatic($name, $arguments){
+        if(is_array($arguments) && count($arguments)>0 && in_array(strtoupper($name), Logger::LEVELS)) {
+            $data = $arguments[0];
+            $name = NULL;
+            $tag = NULL;
+            if (count($arguments)>1) {
+                $name = $arguments[1];
+            }
+            if (count($arguments)>2) {
+                $tag = $arguments[2];
+            }
+            $level = Logger::levelByName($name);
+            return Logger::log($level, $data, $name, $tag);
+        }
+        error_log("Logger::__callStatic($name, \$arguments) => Invalid method name!");
+    }
+
+    public static function levelByName($name) {
+        $val = strtoupper($name);
+        if(in_array($val, Logger::LEVELS)) {
+            return array_search($val, Logger::LEVELS, false);
+        } else {
+            return Logger::FLOG_MAX;
+        }
+    }
+
+    public static function nameByLevel($level) {
+        $level = (int)$level;
+        return Logger::ERROR < $level || $level > Logger::FLOG_MAX ? Logger::LEVELS[Logger::FLOG_MAX] : Logger::LEVELS[$level];
+    }
+
+    public static function log($level, $data = [], $name = NULL, $tag = NULL)
+    {
+        global $savvyext;
+        $clevel = $savvyext->cfgReadLong('phplogger.level');
+        $ilevel = (int)$level;
+        if ($ilevel > $clevel) {
+            error_log("Logger::log() invalid log level! '$level' => '$ilevel' > '$clevel'");
+            return; 
+        }
+        $tag = $tag ?? $savvyext->cfgReadChar('phplogger.tag');
+        $name = $name ?? $savvyext->cfgReadChar('phplogger.name');
+        $data = [
+            "log" => $name,
+            "level" => Logger::nameByLevel($level),
+            "pid" => getmypid(),
+            "zz" => is_array($data) || is_object($data) ? json_encode($data) : $data,
+        ];
+        if (self::getLogger()) {
+            self::getLogger()->post($tag, $data);
+        }
+    }
+
+}
+

CRONS/common/QuoteApi.php

@@ -0,0 +1,74 @@
+<?php
+require_once('../common/Api.php');
+require_once('../common/Logger.php');
+class QuoteApi
+{
+    public static $job_name="";
+    public static function schedule_quote($origin, $destination, $country, $member_id, $transport_provider, $trackedemail_item_id = 0, $prefill = 't', $pool = 1)
+    {
+        if ($origin != "" && $destination != "" && $transport_provider > 0) { // minimal sanity
+            $payload = "{
+                \"origin\":\"${origin}\",
+                \"destination\":\"${destination}\",
+                \"member_id\":${member_id},
+                \"transport_provider_id\":${transport_provider},
+                \"trackedemail_item_id\":${trackedemail_item_id},
+                \"country\":\"${country}\",
+                \"group_quote_id\":0,
+                \"prefill\":\"${prefill}\",
+                \"pool\":${pool}
+            }";
+
+            $input_data = $payload;
+            $endpoint = "/trips/api/quote";
+            $payload = Api::postData($endpoint, $input_data);
+            $log = [
+                'job_name' => self::$job_name,
+                'function' => __FUNCTION__,
+                'request' => $endpoint,
+                'input_data' => $input_data,
+                'response_data' => $payload,
+            ];
+            Logger::debug($log);
+            if (is_array($payload) && array_key_exists('id', $payload) && $payload['id'] > 0) {
+                if (array_key_exists('cost', $payload) && $payload['cost'] > 0) {
+                    return [2, $payload['id']];//completed
+                }
+                if (array_key_exists('travel_date', $payload) && $payload['travel_date'] != '') {
+                    return [-1, $payload['id']];
+                }
+                return [1, $payload['id']];
+            } else {
+                return [-2, null];
+            }
+        }
+        return [-3, null];
+    }
+
+    public static function check_quote($id)
+    {
+        if ($id > 0) { // minimal sanity
+            list($payload, $decrypted, $result, $body) = Api::getData("/trips/api/quote/" . $id);
+            $log = [
+                'job_name' => self::$job_name,
+                'function' => __FUNCTION__,
+                'request' => "/trips/api/quote/" . $id,
+                'input_data' => $id,
+                'response_data' => $payload,
+            ];
+            Logger::debug($log);
+            if (is_array($payload) && array_key_exists('id', $payload) && $payload['id'] > 0) {
+                if (array_key_exists('cost', $payload) && $payload['cost'] > 0) {
+                    return [2, $payload['cost']];//completed
+                }
+                if (array_key_exists('travel_date', $payload) && $payload['travel_date'] != '') {
+                    return [-1, null];
+                }
+            } else {
+                return [-2, null];
+            }
+        }
+        return [-3, null];
+    }
+
+}

CRONS/common/SVY21.php

@@ -0,0 +1,142 @@
+<?php
+/***
+ * Class SVY21
+ * todo: convert SVY21 Singapore TM to Lat Long
+ */
+class SVY21
+{
+    const ELLIPSOID_SEMIMAJORAXIS = 6378137.0;
+    const ELLIPSOID_ECCENTRICITY = 0.0818191908426215;
+    const ELLIPSOID_FLATTENING = 0.00335281066474746;
+    const PROJ_NATURALORIGINLATITUDE = 1.36666666666667;
+    const PROJ_NATURALORIGINLONGITUDE = 103.833333333333;
+    const PROJ_SCALEFACTOR = 1.0;
+    const PROJ_FALSEEASTINGS = 28001.642;
+    const PROJ_FALSENORTHINGS = 38744.572;
+
+    function __construct()
+    {
+    }
+
+    /***
+     * @param $N of svy21
+     * @param $E of svy21
+     * @return float[]|int[] return [lat, long]
+     */
+    public function CnvEN2LL($N, $E)
+    {
+        $l = $this->SecondEccentricity();
+        $a = $this->calc_e1();
+        $b = $this->calc_M1($E);
+        $c = $this->calc_u1($b);
+        $d = $this->calc_lat1($c, $a);
+        $e = $this->calc_T1($d);
+        $f = $this->calc_c1($d);
+        $g = $this->calc_v1($d);
+        $h = $this->calc_p1($d);
+        $i = ($N - self::PROJ_FALSEEASTINGS) / ($g * self::PROJ_SCALEFACTOR);
+        $j = ($i * $i / 2.0) - (5.0 + 3.0 * $e + 10.0 * $f - 4.0 * $f * $f - 9.0 * pow($l, 2)) * (pow($i, 4) / 24.0) + (61.0 + 90.0 * $e + 298.0 * $f + 45.0 * $e * $e - 252.0 * pow($l, 2) - 3.0 * $f * $f) * (pow($i, 6) / 720.0);
+        $j = $d - $g * tan($d) * $j / $h;
+        $lat = $this->CnvRadToDeg($j);
+        $k = ($i - (1.0 + 2.0 * $e + $f) * pow($i, 3) / 6.0 + (5.0 - 2.0 * $f + 28.0 * $e - 3.0 * $f * $f + 8.0 * pow($l, 2) + 24.0 * $e * $e) * pow($i, 5) / 120.0) / cos($d);
+        $k = $this->CnvDegToRad(self::PROJ_NATURALORIGINLONGITUDE) + $k;
+        $long = $this->CnvRadToDeg($k);
+        $r1 = floor($lat);
+        $r2 = floor(($lat - $r1) * 60);
+        $r3 = (round(((($lat - $r1) - ($r2 / 60)) * 60 * 60) * 100) / 100);
+        $r3 = $this->roundNumber($r3, 4);
+        $r4 = floor($long);
+        $r4 = floor(($long - $r4) * 60);
+        $r5 = (round(((($long - $r4) - ($r4 / 60)) * 60 * 60) * 100) / 100);
+        $r5 = $this->roundNumber($r5, 4);
+        return [$lat, $long];
+    }
+
+    private function SecondEccentricity()
+    {
+        return (sqrt(self::ELLIPSOID_ECCENTRICITY * self::ELLIPSOID_ECCENTRICITY / (1.0 - self::ELLIPSOID_ECCENTRICITY * self::ELLIPSOID_ECCENTRICITY)));
+    }
+
+    private function roundNumber($a, $b)
+    {
+        return round($a * pow(10, $b)) / pow(10, $b);
+    }
+
+    private function CnvRadToDeg($a)
+    {
+        return (180.0 * $a / pi());
+    }
+
+    private function CnvDegToRad($a)
+    {
+        return (pi() * $a) / 180.0;
+    }
+
+    private function calc_M($a)
+    {
+        $c = self::ELLIPSOID_ECCENTRICITY;
+        $d = 1.0 - (pow($c, 2) / 4.0) - (3.0 * pow($c, 4) / 64.0) - (5.0 * pow($c, 6) / 256);
+        $e = (3.0 * pow($c, 2) / 8.0) + (3.0 * pow($c, 4) / 32.0) + (45.0 * pow($c, 6) / 1024.0);
+        $f = (15.0 * pow($c, 4.0) / 256.0) + (45.0 * pow($c, 6) / 1024.0);
+        $g = (35.0 * pow($c, 6) / 3072);
+        $b = $d * $a - $e * sin(2.0 * $a) + $f * sin(4.0 * $a) - $g * sin(6 * $a);
+        return $b * self::ELLIPSOID_SEMIMAJORAXIS;
+    }
+
+    private function calc_T1($a)
+    {
+        $b = tan($a);
+        return $b * $b;
+    }
+
+    private function calc_v1($a)
+    {
+        $b = self::ELLIPSOID_ECCENTRICITY * sin($a);
+        $b = sqrt(1.0 - $b * $b);
+        return self::ELLIPSOID_SEMIMAJORAXIS / $b;
+    }
+
+    private function calc_p1($a)
+    {
+        $c = self::ELLIPSOID_ECCENTRICITY;
+        $b = 1.0 - $c * $c * pow(sin($a), 2);
+        $b = pow($b, 3.0 / 2.0);
+        return self::ELLIPSOID_SEMIMAJORAXIS * (1.0 - $c * $c) / $b;
+    }
+
+    private function calc_c1($a)
+    {
+        $b = $this->SecondEccentricity();
+        $b = $b * cos($a);
+        return $b * $b;
+    }
+
+    private function calc_e1()
+    {
+        $a = 1.0 - sqrt(1.0 - pow(self::ELLIPSOID_ECCENTRICITY, 2));
+        return $a / (1.0 + sqrt(1.0 - pow(self::ELLIPSOID_ECCENTRICITY, 2)));
+    }
+
+    private function calc_M1($a)
+    {
+        $c = $this->calc_M($this->CnvDegToRad(self::PROJ_NATURALORIGINLATITUDE));
+        return ($c + ($a - self::PROJ_FALSENORTHINGS) / self::PROJ_SCALEFACTOR);
+    }
+
+    private function calc_u1($a)
+    {
+        $c = self::ELLIPSOID_ECCENTRICITY;
+        $b = self::ELLIPSOID_SEMIMAJORAXIS * (1.0 - ($c * $c / 4.0) - (3.0 * pow($c, 4) / 64.0) - (5.0 * pow($c, 6) / 256.0));
+        return $a / $b;
+    }
+
+    private function calc_lat1($a, $b)
+    {
+        $c = $a + ((3.0 * $b / 2.0) - 27.0 * $b * $b / 32.0) * sin(2.0 * $a);
+        $c = $c + ((21.0 * $b * $b / 16.0) - 55.0 * pow($b, 4.0) / 32.0) * sin(4.0 * $a);
+        $c = $c + (151.0 * pow($b, 3) / 96.0) * sin(6.0 * $a);
+        $c = $c + (1097.0 * pow($b, 4) / 512.0) * sin(8.0 * $a);
+        return $c;
+    }
+
+}

CRONS/common/address_city.php

@@ -0,0 +1,449 @@
+<?php
+
+function getAddressId($address)
+{
+    global $googleKey, $readOnlyReplicaConn;
+    // Check the address exsts
+    $q = "SELECT id FROM address WHERE lower(address)=lower('${address}') ";
+    $q .= "AND latitude<>0 AND longitude<>0 AND geocoding_date IS NOT NULL ";
+    $q .= "ORDER BY geocoding_date DESC LIMIT 1";
+    error_log($q);
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return null;
+}
+
+/* save data to table address */
+function saveAddress($result)
+{
+    global $conn;
+
+    syslog(LOG_WARNING, 'MessageDataSaver::saveAddress($result)');
+    $db_lat = floatval($result["lat"]);
+    $db_lng = floatval($result["lng"]);
+    $db_address = pg_escape_string($result["address"]);
+    $db_postal = pg_escape_string($result["postal"]);
+    $db_city = pg_escape_string($result["city"]);
+    $db_city_lat = floatval($result["city_lat"]);
+    $db_city_lng = floatval($result["city_lng"]);
+    $db_country = pg_escape_string($result["country"]);
+    $db_tz = getTimeZoneID($result["timeZoneId"]);
+    $city_id = getCityId($db_city, $db_country, $db_city_lat, $db_city_lng);
+    if (empty($db_postal)) {
+        $db_postal = getPostalCode($db_lat, $db_lng, $db_country);
+    }
+    $q = "INSERT INTO address (address,latitude,longitude,timezone,geocoding_date,postal,country,geometry, city_id) VALUES (";
+    $q .= "'${db_address}',${db_lat},${db_lng}," . ($db_tz == null ? "NULL" : $db_tz) . ",now(),'${db_postal}','${db_country}',ST_SetSRID(ST_MakePoint(${db_lng},${db_lat}), 4326),'${city_id}')";
+    $q .= " RETURNING id";
+    error_log($q);
+    $r = pg_query($conn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return null;
+}
+
+function getTzByAddressId($id)
+{
+    global $readOnlyReplicaConn;
+
+    $q = "SELECT timezone FROM address_timezone WHERE id=(SELECT timezone FROM address WHERE id=" . ((int) $id) . ")";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return 'America/Los_Angeles'; // 2
+}
+
+function getTimeZoneID($name)
+{
+    global $readOnlyReplicaConn, $conn;
+
+    $q = "SELECT id FROM address_timezone WHERE timezone='" . pg_escape_string($name) . "'";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    $q = "INSERT INTO address_timezone (timezone) VALUES('" . pg_escape_string($name) . "') RETURNING id";
+    $r = pg_query($conn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return null;
+}
+
+function getCityId($city, $country_code, $latitude, $longitude)
+{
+    global $readOnlyReplicaConn, $conn;
+    $q = "SELECT id  FROM geofence_area_city WHERE LOWER(city) ILIKE '" . strtolower(pg_escape_string($city)) . "' AND country='" . $country_code . "' OR (latitude='" . $latitude . "' AND longitude='" . $longitude . "') LIMIT 1";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_assoc($r)) {
+        return $f['id'];
+    }
+    $i = "INSERT INTO geofence_area_city (city,country,latitude,longitude,location) VALUES ('" . pg_escape_string($city) . "','${country_code}',${latitude},${longitude},ST_SetSRID(ST_MakePoint(${longitude},${latitude}), 4326)) RETURNING id";
+    $r = pg_query($conn, $i);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return null;
+}
+
+function deleteCities($ids)
+{
+    global $conn;
+    if (is_array($ids) && count($ids) > 0) {
+        $q1 = "DELETE FROM geofence_area_city WHERE  id IN (" . implode(",", $ids) . ")";
+        $r1 = pg_query($conn, $q1);
+        if ($r1 && pg_affected_rows($r1)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+function deleteDataTable($table_name, $condition = "")
+{
+    global $conn;
+    if (!empty($condition)) {
+        $q1 = "DELETE FROM " . $table_name . " WHERE  " . $condition;
+        $r1 = pg_query($conn, $q1);
+        if ($r1 && pg_affected_rows($r1)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+function updateDataTable($table_name, $condition, $data)
+{
+    global $conn, $readOnlyReplicaConn;
+    $q = "select column_name, data_type, character_maximum_length from INFORMATION_SCHEMA.COLUMNS where table_name = '" . $table_name . "'";
+
+    $r = pg_query($readOnlyReplicaConn, $q);
+    while ($f = pg_fetch_assoc($r)) {
+        $columns[$f['column_name']] = $f['data_type'];
+    }
+    $q = "UPDATE " . $table_name . " SET id=id";
+    foreach ($data as $key => $val) {
+        if ($key == "id") {
+            continue;
+        }
+        if (array_key_exists($key, $data) && !in_array($columns[$key], ['character varying', 'date', 'timestamp with time zone', 'timestamp without time zone'])) {
+            $q .= ", ${key} = " . ($val == "" ? "NULL" : $val);
+        } else {
+            $q .= ", ${key} = '" . pg_escape_string($val) . "'";
+        }
+    }
+    $q .= " WHERE " . $condition . "";
+    echo $q . "\n";
+    $r = pg_query($conn, $q);
+
+    $updated = pg_affected_rows($r);
+    echo "[" . date("Y-m-d H:i:s") . "]  update  {$updated} rows of {$table_name} with {$condition} .\n";
+    return $updated;
+}
+
+function getForeignTables($table_name)
+{
+    global $readOnlyReplicaConn;
+    $q = "SELECT
+            tc.table_name, kcu.column_name,
+            ccu.table_name AS foreign_table_name,
+            ccu.column_name AS foreign_column_name
+        FROM
+            information_schema.table_constraints AS tc
+            JOIN information_schema.key_column_usage AS kcu
+            ON tc.constraint_name = kcu.constraint_name
+            JOIN information_schema.constraint_column_usage AS ccu
+            ON ccu.constraint_name = tc.constraint_name
+        WHERE constraint_type = 'FOREIGN KEY' AND ccu.table_name='" . $table_name . "'";
+    $r = pg_query($readOnlyReplicaConn, $q);
+
+    $result = [];
+    while ($f = pg_fetch_assoc($r)) {
+        $result[] = $f;
+    }
+    return $result;
+}
+
+function getGeoName($city, $country_code)
+{
+    global $readOnlyReplicaConn;
+    $q = "SELECT name,country,latitude,longitude,location FROM geoname WHERE LOWER(name) ILIKE '" . strtolower(pg_escape_string($city)) . "' AND country='" . $country_code . "' LIMIT 1";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_assoc($r)) {
+        return $f;
+    } else {
+        return null;
+    }
+}
+
+function insertPostalCode($geoname, $postal, $latitude, $longitude, $country)
+{
+    global $conn, $readOnlyReplicaConn;
+
+    if (strlen($geoname) > 100) {
+        $geoname = substr($geoname, 0, 99);
+    }
+    $q = "SELECT id  FROM geoname_postal_code WHERE postal ILIKE '" . pg_escape_string($postal) . "' AND  latitude='" . $latitude . "' AND longitude='" . $longitude . "' LIMIT 1";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_assoc($r)) {
+        return $f['id'];
+    }
+    $i = "INSERT INTO geoname_postal_code (geoname, postal, latitude,longitude, country, geometry) VALUES ('" . pg_escape_string($geoname) . "','" . pg_escape_string($postal) . "',${latitude},${longitude},'" . pg_escape_string($country) . "',ST_SetSRID(ST_MakePoint(${longitude},${latitude}), 4326)) RETURNING id";
+    $r = pg_query($conn, $i);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_row($r)) {
+        return $f[0];
+    }
+    return null;
+}
+
+function getPostalCode($latitude, $longitude, $country, $distance = 50000)
+{
+    global $readOnlyReplicaConn;
+
+    $q = "SELECT postal,
+    ST_DistanceSphere(geometry::geometry, ST_SetSRID(ST_MakePoint(" . $longitude . ", " . $latitude . "),4326)) AS distance
+      FROM geoname_postal_code
+      WHERE ST_DistanceSphere(geometry::geometry, ST_SetSRID(ST_MakePoint(" . $longitude . ", " . $latitude . "),4326)) BETWEEN 0 AND " . $distance . " AND country='" . $country . "'
+      ORDER BY distance
+      LIMIT 1;";
+    $r = pg_query($readOnlyReplicaConn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_assoc($r)) {
+        return $f['postal'];
+    }
+
+    return null;
+}
+
+/**
+ * get postal array from latitude, longitude
+ */
+function getPostalFromLatLng($latitude, $longitude, int $distance_in_km)
+{
+    global $conn, $readOnlyReplicaConn;
+    $earth_radius = 6371;
+    $km_per_degree_lat = 111.2;
+    $pi = 3.14 / 180;
+
+    $query = " SELECT postal FROM (
+            SELECT (
+                $earth_radius * acos(
+                cos( radians( {$latitude}) )
+                * cos( radians( latitude  ) )
+                * cos( radians( longitude ) - radians( {$longitude} ) )
+                + sin( radians( {$latitude}) )
+                * sin( radians( latitude  ) )
+            )
+            ) AS distance
+            , *
+            FROM postal_code
+            WHERE
+            postal IS NOT NULL
+            AND latitude BETWEEN " . ($latitude - ($distance_in_km / $km_per_degree_lat)) .
+        " AND " . ($latitude + ($distance_in_km / $km_per_degree_lat));
+
+    $delta = round($distance_in_km / ($km_per_degree_lat * COS(deg2rad($longitude))), 10);
+
+    // Bounding box for speed - latitude within range (longtitude to km not linear)
+    if (cos($longitude * $pi) > 0) {
+
+        $from = $longitude - $delta;
+        $to = $longitude + $delta;
+
+        $query .=
+            " AND address.longitude" .
+            " BETWEEN " . $from .
+            " AND " . $to;
+
+    } else {
+
+        $from = $longitude + $delta;
+        $to = $longitude - $delta;
+
+        $query .=
+            " AND address.longitude" .
+            " BETWEEN " . $from .
+            " AND " . $to;
+
+    }
+
+    // distance limit for circle
+    $query .= " ) address
+            WHERE distance < " . $distance_in_km;
+
+    $rs = pg_query($readOnlyReplicaConn, $query) or die("Cannot execute query: $query\n");
+
+    if (!$rs) {
+        echo "An error occurred.\n";
+        exit;
+    }
+
+    return array_column(pg_fetch_all($rs), 'postal');
+}
+
+function insertActionLog($table_name="quotes",$operation="",$input_data="", $response_data="", $url="")
+{
+    global $conn;
+
+    $input_data = is_array($input_data) || is_object($input_data) ? json_encode($input_data) : $input_data;
+    $response_data = is_array($response_data) || is_object($response_data) ? json_encode($response_data) : $response_data;
+    $q = "INSERT INTO action_logs (created,table_name,operation,input_data,response_data, url) ";
+    $q.= " VALUES(NOW(),'".$table_name."','".$operation."',";
+    $q.= "'".pg_escape_string($input_data)."','".pg_escape_string($response_data)."','".$url."') RETURNING id";
+    $r = pg_query($conn, $q);
+    if ($r && pg_num_rows($r) && $f=pg_fetch_assoc($r)) {
+        return true;
+    }
+    return null;
+}
+
+function parseAddressByCountry($address = "", $country_code = "")
+{
+    $city = $state = $country = "";
+    if ($country_code == "AE") { //United Arab Emirates
+        $address_info = explode("-", $address);
+    } else {
+        $address_info = explode(",", $address);
+    }
+
+    switch ($country_code) {
+        case "AU": //Australia
+            $country = trim(array_pop($address_info));
+            $state_city = explode(" ", trim(array_pop($address_info)));
+            $postal = trim(array_pop($state_city));
+            $state = trim(array_pop($state_city));
+            $city = trim(array_pop($state_city));
+            break;
+        case "BR": //Brazil
+            $country = trim(array_pop($address_info));
+            $state = trim(array_pop($address_info));
+            $city = array_pop($address_info);
+            $city = explode('-', $city);
+            $city = trim($city[0]);
+            break;
+        case "SG":
+            $country = $city = "Singapore";
+            break;
+        case "HK":
+            $country = $city = "Hong Kong";
+            break;
+        case "US":
+            $country = trim(array_pop($address_info));
+            $state = trim(array_pop($address_info));
+            $city = $state;
+            if (preg_replace('/[^0-9]/', '', $state) != "") {
+                $city = trim(array_pop($address_info));
+            }
+            break;
+        case "CA": //Canada
+        case "IN": //India
+        case "ZA": //South Africa
+        case "MY": //Malaysia
+        case "PH": //Philippines
+            $country = trim(array_pop($address_info));
+            $state = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            if ($country_code == "MY") {
+                $city = parseName($city);
+            }
+            break;
+        case "ID": //Indonesia
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            $city = parseName($city);
+            $city_info = explode(" ", $city);
+            $city = trim(array_pop($city_info));
+            break;
+        case "AT": //Autria
+        case "FI": //Finland
+        case "SE": //Sweden
+        case "BE": //Belgium
+        case "DE": //Germany
+        case "FR": //France
+        case "TH": //Thailand
+        case "NP": //Nepal
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            $city = parseName($city);
+            break;
+
+        case "NZ": //New Zealand
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            $city = parseName($city);
+            if ($city == "") {
+                $city = trim(array_pop($address_info));
+            }
+            break;
+
+        case "NL": //Netherlands
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            $city = parseName($city);
+            $city = explode(' ', $city);
+            $city = trim(array_pop($city));
+            break;
+
+        /*case "TR"://Turkey
+        $country = trim(array_pop($address_info));
+        $city = array_pop($address_info);
+        $city = explode('/', $city);
+        $city = trim(array_pop($city));
+        break;*/
+        case "UA": //Ukraina
+            $count_address = count($address_info);
+
+            if ($count_address < 5) {
+                $country = trim(array_pop($address_info));
+                $city = trim(array_pop($address_info));
+            } else {
+                if ($count_address == 5) {
+                    $country = $address_info[$count_address - 2];
+                    $city = $address_info[$count_address - 3];
+                } else {
+                    $country = $address_info[$count_address - 2];
+                    $city = $address_info[$count_address - 4];
+                }
+            }
+            $city = parseName($city);
+            break;
+        case "GB": //Great Britain
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+            $city = explode(' ', $city);
+            $city = trim($city[0]);
+            break;
+
+        case "UA": //Ukraina
+        case "TR": //Turkey
+        case "PL": //Poland
+        case "RO": //Romania
+            $city = "";
+            break;
+        case "TW": //Taiwan
+            $country = trim(array_pop($address_info));
+            $city = array_pop($address_info);
+            break;
+        default:
+            $country = trim(array_pop($address_info));
+            $city = trim(array_pop($address_info));
+    }
+
+    return [
+        "address" => $address,
+        "city" => $city,
+        "state" => $state,
+        "country_code" => $country_code,
+        "country" => $country,
+    ];
+
+}
+
+function parseName($name)
+{
+    $name = trim(preg_replace('/[0-9]+/', '', $name));
+    return $name;
+}

CRONS/common/config.php

@@ -0,0 +1,25 @@
+<?php
+require '../backend.php';
+
+$httpAuthToken = $savvyext->cfgReadChar('system.oauth2_token');
+$oauth_api = $savvyext->cfgReadChar('system.oauth2_url');
+
+$db_host = $savvyext->cfgReadChar('database.host');
+$db_name = $savvyext->cfgReadChar('database.name');
+$db_user = $savvyext->cfgReadChar('database.user');
+$db_pass = $savvyext->cfgReadChar('database.pass');
+$db_port = $savvyext->cfgReadLong('database.port');
+$connstr = "host=${db_host} port=${db_port} dbname=${db_name} user=${db_user} password=${db_pass}";
+$conn = pg_connect($connstr);
+
+// Create readOnlyReplicaConn for readonly database
+$db_read_only_host = $savvyext->cfgReadChar('database_replica.host');
+$db_read_only_name = $savvyext->cfgReadChar('database_replica.name');
+$db_read_only_user = $savvyext->cfgReadChar('database_replica.user');
+$db_read_only_pass = $savvyext->cfgReadChar('database_replica.pass');
+$db_read_only_port = $savvyext->cfgReadLong('database_replica.port');
+$readonlyconnstr = "host=${db_read_only_host} port=${db_read_only_port} dbname=${db_read_only_name} user=${db_read_only_user} password=${db_read_only_pass}";
+$readOnlyReplicaConn = pg_connect($readonlyconnstr);
+
+$city_keywords = ["city", "province", "capital", "town"];
+

CRONS/common/parser.php

@@ -0,0 +1,38 @@
+<?php
+
+function parseEmail($email_id)
+{
+    global $httpAuthToken,$oauth_api;
+    $url = $oauth_api."parse/" . $email_id;
+    $opts = array(
+        'http' => array(
+            'method' => "GET",
+            'timeout' => 60, /* 1 minute */
+            'header' =>
+            "Content-Type: application/x-www-form-urlencoded\r\n" .
+            "Accept: application/json\r\n" .
+            "Authorization: Server-Token ${httpAuthToken}\r\n",
+        ),
+    );
+
+    $context = stream_context_create($opts);
+    $body = file_get_contents($url, false, $context);
+    $geocoded = json_decode($body, true);
+    if (is_array($geocoded) && is_array($geocoded["data"]) && !isset($geocoded["error"])) {
+        return $geocoded["data"];
+    } 
+    return null;
+}
+
+
+function updateParsedEmail($id, $location_start_id, $location_end_id)
+{
+    global $conn;
+    $q = "UPDATE parsedemail_item SET updated=NOW(), location_start_id=" . $location_start_id . ", location_end_id=" . $location_end_id . "
+    WHERE id = " . $id . "";
+    $r = pg_query($conn, $q);
+    if ($r && pg_num_rows($r) && $f = pg_fetch_assoc($r)) {
+        return [$f, null];
+    }
+    return [null, pg_last_error($conn)];
+}

CRONS/common/utils.php

@@ -0,0 +1,33 @@
+<?php
+
+function d($value, $exit = 0)
+{
+    if (is_array($value)) {
+        print_r($value);
+    } else if (is_object($value)) {
+        var_dump($value);
+    } else {
+        echo $value;
+    }
+    echo "<br/>";
+    if ($exit) {
+        exit;
+    }
+}
+
+function convertUtcToLocal($datetime, $timezone = '', $format = 'Y-m-d H:i:s')
+{
+    if (!empty($datetime) && !empty($timezone)) {
+        $datetime = date($format, strtotime($datetime));
+        $utc_date = DateTime::createFromFormat(
+            $format,
+            $datetime,
+            new DateTimeZone('UTC')
+        );
+        if ($utc_date) {
+            $utc_date->setTimeZone(new DateTimeZone($timezone));
+            return $utc_date->format($format);
+        }
+    }
+    return $datetime;
+}