#!/usr/bin/perl
use strict;
use warnings FATAL => 'all';
use File::Slurp;
use File::Copy;
use IO::File;
use JSON::PP;
use Data::Dumper;
use MIME::Base64;
use Gzip::Faster qw( inflate_raw );
use Gearman::Worker;
use Gearman::Client;
use Storable qw( thaw );
use DBI;
use DBIx::Dump;
use Digest::MD5 qw(md5_hex);
use Regexp::Common qw(time);
use Syntax::Keyword::Try;


my $dateTimePattern = '%Y-%m-%d %H:%M:%S';
my $tracklocation_dir = $ENV{TRACKLOCATION_DIR} or die "Environment variable TRACKLOCATION_DIR not specified";
if (('/' ne substr($tracklocation_dir, -length('/')))) {
    $tracklocation_dir .= '/';
}

my $dbh = DBI->connect ("dbi:CSV:", "", "", {
    f_dir => $tracklocation_dir,
    csv_sep_char => "|"
});

my $client = Gearman::Client->new;
$client->job_servers('127.0.0.1');

my $worker = Gearman::Worker->new;
$worker->job_servers('127.0.0.1');
$worker->register_function(tl_decompress => \&decompress);
$worker->work while 1;


sub decompress {
    my $job = shift;
    my $workload = $job->arg;
    my $data = decode_json($workload);
    my $tlSession = $data->{tl_session};

    print STDERR "[".localtime()."] Start process [$tlSession]\n";

    my @files = read_dir($tracklocation_dir);
    my @filesFiltered = grep /^(tl_(\d)*\_$tlSession\_(\w|\d|\.)*\.json.lock)$/, @files;

    my $length  = scalar @filesFiltered;
    if ($length == 0 ) {
        print STDERR "[".localtime()."] End process [$tlSession]\n";
        return;
    }

    my $deviceToken="";
    my $loc="";

    my %sortedFiles = ();
    foreach(@filesFiltered) {
        if ($_ =~ /tl_(\d)*\_$tlSession\_(\w+.\w+)\.json/) {
            $sortedFiles{$_} = $2 + 0;
        }
    }
    my $column = "traked_date|traked_group|member_id|latitude|longitude|speed|timestamp|point_count|tmp\n";

    my (
        $curline
        , $fh
        , $fh_in
        , $fh_out
        , $dict
        , $key
        , $count
        , $isValid
    );
    my $tmpFileName = lc("tl_tmp_$tlSession.csv");
    my $csvFileName = $tracklocation_dir.$tmpFileName;
    my $tmp_file = $tracklocation_dir.lc("tmp_$tlSession.csv");
    try {
        open($fh, '>', $csvFileName) or die "Can't open '$csvFileName' !";
        print $fh $column;
        close $fh;
        open($fh, '>>', $csvFileName) or die "Can't open '$csvFileName' !";
        foreach my $file (sort { $sortedFiles{$a} <=> $sortedFiles{$b} } keys %sortedFiles) {
            if (-d  $tracklocation_dir."debug") {
                copy $tracklocation_dir.$file, $tracklocation_dir."debug";
            }
            my $json_text = read_file($tracklocation_dir.$file);
            my $data;

            try {
                $data = decode_json($json_text);
            } catch {
                print STDERR "[".localtime()."] Could not decode JSON from file: '".$tracklocation_dir.$file."'\n";
            };
            if (!$data) {
                next;
            }
            if(!defined($deviceToken) || !length($deviceToken)) {
                $deviceToken = $data->{deviceToken};
            }
            if(!defined($loc) || !length($loc)) {
                $loc = $data->{loc};
            }
            # my $b64decoded = decode_base64($data->{batch});
            # print $fh inflate_raw($b64decoded);

            open($fh_out, '>',  $tmp_file);
            my $decoded = decode_base64($data->{batch});
            try {
                print $fh_out inflate_raw($decoded);
            } catch {
                print STDERR "[".localtime()."] Could not inflate decoded data. Maybe data is not compressed. \n";
                print $fh_out $decoded;
            };
            # print $fh_out inflate_raw(decode_base64($data->{batch}));
            close $fh_out;
            $fh_in  = new IO::File("< $tmp_file");
            while (<$fh_in>) {
                chomp;
                $curline = $_;
                $isValid = $curline =~ m/^$RE{time}{strftime}{-pat => $dateTimePattern}/;
                if ($isValid) {
                    $count = () = $curline =~ /\Q|/g;
                    if (8 != $count) {
                        next;
                    }
                } else {
                    next;
                }
                $key = md5_hex($curline);
                if (!exists($$dict{$key})) {
                    $$dict{$key} = 1;
                    print $fh $curline."\n";
                }
            }
        }
        close $fh;

        if (-d  $tracklocation_dir."debug") {
            copy $csvFileName, $tracklocation_dir."debug";
        }
        my $tlFilePath = $tracklocation_dir."tl_$tlSession.csv";
        try {
            my $query = "SELECT DISTINCT traked_date,traked_group,member_id,latitude,longitude,speed,timestamp,point_count,tmp FROM $tmpFileName";
            my $sth   = $dbh->prepare ($query);
            $sth->execute ();

            my $out = DBIx::Dump->new('format' => 'csv', # excel or csv
                'output' => $tlFilePath, # file to save as
                'sth' => $sth);
            $out->dump();
            $sth->finish ();
        } catch {
            unlink  $tlFilePath;
            open($fh, '>', $tlFilePath) or die "Can't open '$tlFilePath' !";
            print $fh "TRAKED_DATE,TRAKED_GROUP,MEMBER_ID,LATITUDE,LONGITUDE,SPEED,TIMESTAMP,POINT_COUNT,TMP";
            close $fh;
        };

        if (-d  $tracklocation_dir."debug") {
            copy $tracklocation_dir."tl_$tlSession.csv", $tracklocation_dir."debug";
        }
    } catch {
        print STDERR "[".localtime()."] Something gone wrong. \n";
        print STDERR "[".localtime()."] $@ \n";
    } finally {
        if (fileno($fh)) {
            try {
                close $fh;
            } catch {
            }
        }
        if (fileno($fh_in)) {
            try {
                close $fh_in;
            } catch {
            }
        }
        if (fileno($fh_out)) {
            try {
                close $fh_out;
            } catch {
            }
        }
        try {
            unlink $tmp_file;
            unlink $csvFileName;
            unlink $tracklocation_dir.$tmpFileName;
        } catch {
        }
    };

    my %payload = (
        'deviceToken' => $deviceToken,
        'loc'         => $loc,
        'sessionId'   => $tlSession
    );

    my $arg_p = encode_json(\%payload);
    my $argref = ref $arg_p ? $arg_p : \$arg_p;
    try {
        my $task = Gearman::Task->new("process_tl_batch", $argref, {on_fail => sub { print STDERR "$@", "\n"; },});
        $client->dispatch_background($task);
    } catch {
        print STDERR "[".localtime()."] Run job 'process_tl_batch' failed. \n";
        print STDERR "[".localtime()."] $@ \n";
    };

    print STDERR "[".localtime()."] End process [$tlSession]\n";
    return;
}


