#!/usr/bin/perl


use strict;
#use utf8;
#use warnings;
#use feature 'say';
#use open qw/ :std OUT :encoding(UTF-8) /;
#use warnings;
#use BerkeleyDB;
use DB_File;
use Date::Parse;
use Encode qw(decode encode);
#use DateTime;
use Fcntl ':flock';
use v5.10;
use Text::Wrap;
use XML::FeedPP ();
use FTN::Packet qw(&write_ftn_packet);
#use FTN::Packet;
use FTN::Log qw(&logging);


my $res1 = 'https://www.pravda.com.ua/rus/rss/';
#my $res2 = 'https://www.obozrevatel.com/out/rss/lastnews.xml';
my $res3 = 'https://focus.ua/modules/rss.php';
my $res4 = 'https://theins.ru/feed';
my $res5 = 'https://www.pravda.com.ua/eng/rss-info/';
my $res6 = 'https://gazeta.ua/ru/rss';
my $feed = XML::FeedPP->new( $res1 );

my %database;
my %database2;
my %packet_info;
my @messages;

my $db = tie %database, 'DB_File', "/data/ftp/pub/CODE/FIDO/rssbot/PRAVDADB.dat", O_CREAT | O_RDWR, 0666 or die "Cannot initial: $! \n ";
my $fd = $db->fd();
my $db2 = tie %database2, 'DB_File', "/data/ftp/pub/CODE/FIDO/rssbot/PRAVDADB2.dat", O_CREAT | O_RDWR, 0666 or die "Cannot initial: $! \n ";
my $fd2 = $db2->fd();
open DATAFILE, "+<&=$fd" or "Impossible safe open db: $! \n";
flock(DATAFILE, LOCK_SH) or die "Cannot block: $! \n";
open DATAFILE2, "+<&=$fd2" or "Impossible safe open db: $! \n";
flock(DATAFILE2, LOCK_SH) or die "Cannot block: $! \n";
foreach my $item ( $feed->get_item() ) {
#
    my $title1 = $item->title();
    my $desc1 = $item->description();
    my $link1 = $item->link();
    my $pbdt1 = $item->pubDate();
# Decode to CP866
#    $title1 = encode("cp866", $title1);
#    $desc1 = encode("cp866", $desc1);
my $unixtime = str2time($pbdt1);
if(exists($database2{$unixtime}))
    {
        # nothing    
        }
        else
         {
	      chomp $title1;
	      chomp $desc1;
	      chomp $link1;
	      chomp $pbdt1;
	     #  next if($title1 =~ s/^\s*$/);
             #  print "111\n";    
	     #  next if($desc1 =~ s/^\s*$/);
	     #  print "222\n";	
             #  next if($link11 =~ s/^\s*$/);
	     #  print "333\n";
             #  next if($pbdt1 =~ s/^\s*$/);
	     #  print "444\n";
		#   $title1=~ s{ \s* \z}{}xms;
	        #   $desc1=~ s{ \s* \z}{}xms;
		#   $link1=~ s{ \s* \z}{}xms;
		#   $pbdt1=~ s{ \s* \z}{}xms;
		##   $title1=~s/\r?\n\z//;
                #   $desc1=~s/\r?\n\z//;
                #   $link1=~s/\r?\n\z//;
                #   $pbdt1=~s/\r?\n\z//;
		#   $title1=~s/\r//g;
		#   $desc1=~s/\r//g;
		   $title1=~s/\s+/ /g;
		   $desc1=~s/\s+/ /g;
		   $title1=~s/<(?:[^>'"]*|(['"]).*?\1)*>//gs;
		   $desc1=~s/<(?:[^>'"]*|(['"]).*?\1)*>//gs;
                   my $datadb = $title1."||".$desc1."||".$link1."||".$pbdt1;
                   $database{$unixtime}=$datadb;
          }
 }
undef $db;
untie %database;
close DATAFILE;


undef $db2;
untie %database2;
close DATAFILE2;

undef %database;
undef %database2;
undef $fd;
undef $fd2;
my $idx;
undef my %onedb;
#undef my $key;
undef my $title1;
undef my $desc1;
undef my $link1;
undef my $pbdt1;

# all news records

my %database;
# already send
my %database2;
my $db = tie %database, 'DB_File', "/data/ftp/pub/CODE/FIDO/rssbot/PRAVDADB.dat", O_CREAT | O_RDWR, 0666 or die "Cannot initial: $! \n ";
my $db2 = tie %database2, 'DB_File', "/data/ftp/pub/CODE/FIDO/rssbot/PRAVDADB2.dat", O_CREAT | O_RDWR, 0666 or die "Cannot initial: $! \n ";

my $fd = $db->fd();
my $fd2 = $db2->fd();

open DATAFILE, "+<&=$fd" or "Impossible safe open db: $! \n";
#flock(DATAFILE, LOCK_SH) or die "Cannot block: $! \n";
open DATAFILE2, "+<&=$fd2" or "Impossible safe open db: $! \n";
#flock(DATAFILE2, LOCK_SH) or die "Cannot block: $! \n";

$idx=0;
my %onedb;

foreach my $key (sort keys %database)
    {
#print "$key \n", ("="x(length($key)+1)),"\n\n";
my ($title, $desc, $link, $pabdate) = split(/\|\|/, $database{$key} );
#my $datastr=join(/\|\|/,$title,$desc,$link,$pabdate);

my $datastr=$title . "||" . $desc . "||" . $link . "||" . $pabdate;

    if($idx<=0)
        {
            $onedb{$key}=$datastr;
            $database2{$key}=$datastr;

        }
    if($idx<=0)
        {
        delete $database{$key};
        }

    $idx++;
}

undef $db;
untie %database;
close DATAFILE;

undef $db2;
untie %database2;
close DATAFILE2;

undef %database;
undef %database2;
undef $fd;
undef $fd2;
undef my $fidotitl;
undef my $kkk;
undef my $title1;
undef my $desc1;
undef my $link1;
undef my $pabdate1;

my $fidotitl;
my $tui=0;

open(FILEHANDLE,">/data/ftp/pub/CODE/FIDO/rssbot/pravda.dat") or die ("Cannot open file");

foreach my $kkk (sort keys %onedb)
{

my ($title1,$desc1,$link1,$pabdate1)=split(/\|\|/, $onedb{$kkk} );
#$title1 = encode("cp866", $title1);
#$desc1 = encode("cp866", $desc1);
#$pabdate1 = encode("cp866", $pabdate1);

$Text::Wrap::columns=70;
$Text::Wrap::separator="\n";
$desc1=wrap("","",$desc1);
$title1=wrap("","",$title1);

#print FILEHANDLE " \n";
print FILEHANDLE $title1, "\n";
if($tui==0) { $fidotitl=$title1; }
print FILEHANDLE $desc1, "\n";
print FILEHANDLE $link1, "\n";
print FILEHANDLE $pabdate1, "\n";
print FILEHANDLE "\n";
$tui++;

}
close FILEHANDLE;

my $texxt;
my $newtexxt;

open(FILEH,"</data/ftp/pub/CODE/FIDO/rssbot/pravda.dat") or die ("Cannot open file");
$texxt="";
while (read (FILEH, $newtexxt, 1))
{
$texxt.=$newtexxt;
}
#chomp(@array=<FILEH>);

close FILEH;



my $spool_dir = "/data/ftp/pub/CODE/FIDO/rssbot";
my $OutDir = "$spool_dir/inbound";
my $log_file = "$spool_dir/packet.log";

my $ftnsubj=substr($fidotitl, 0, 70);
#my $ftnbody=join('\n ' @array);
my $ftnbody=$texxt;

${messages}[0]{area}="4.PVT";
${messages}[0]{tonode}="2:467/4";
${messages}[0]{Fromnode}="2:467/4";
${messages}[0]{From}="Novyny Ukraini";
${messages}[0]{To}="All";
${messages}[0]{Subj}=$ftnsubj;
${messages}[0]{Body}=$ftnbody;
${messages}[0]{ctrlinfo}="TID: FIDOGATE-5.12-a97f4004";
${packet_info}{origNet}="467";
${packet_info}{PassWord}="TRF5UAUA";                             # a8
${packet_info}{origZone}="2";                              # S   (repeat)
${packet_info}{destZone}="2";                             # S   (repeat)
${packet_info}{origPoint}="444";
${packet_info}{origNode}="4";                          # S   (repeat)
${packet_info}{destNode}="4";                         # S   (repeat)
${packet_info}{destNet}="467";
# Time format: 17 Oct 22 23:19:06
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
$year += 1900;
my @abbr = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
#$year = substr($year,2);
my $ttime=$mday." ".$abbr[$mon]." ".$year." ".$hour.":".$min.":".$sec;
#
#my $ttime=localtime();

${messages}[0]{ftscdate}=$ttime;
${messages}[0]{DateTime}=$ttime;
${packet_info}{Area}="4.PVT";
${packet_info}{Origin}=" /RoboNews v.1.3 Novyny Ukraini/ ";
${packet_info}{TearLine}=" *Read us with http://winpoint.org* ";

write_ftn_packet($OutDir, \%packet_info, \@messages);
