OXIESEC PANEL

Name	Size	Modified	Perms
📄 -d	10 bytes	02/03/2023 07:34:13 AM	rw-r--r--
📁 ..	-	02/03/2023 08:06:37 AM	rwxr-xr-x
📄 config.pl	7.61 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 cross2normal.pl	3.71 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 dataextract.sh	2.51 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 etl.pl	5.42 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 etlproc	9.99 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 func.pl	4.01 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 globvar.sh	127 bytes	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 initetl.pl	1.88 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 libawk.awk	3.07 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 makeXtab2NormalExcel.pl	5.73 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 process.pl	6.91 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 process_changed.pl	5.69 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 readwrite.pl	4.16 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxcsv.pl	7.94 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxdb.pl	11.06 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxfile.pl	4.68 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxsalesforce.pl	3.5 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxxml.pl	13.83 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 rxxml2.pl	4.18 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 sendmail.pl	1.06 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 sourcequery.pl	4.18 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 sql.pl	23.98 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 store.pl	7.97 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 tags	13.14 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 task.pl	33.48 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📁 test	-	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 userfunc.pl	5.05 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 valid.pl	3.64 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 workflow.pl	3.94 KB	02/03/2023 07:34:13 AM	rwxr-xr-x
📄 xform.pl	2.62 KB	02/03/2023 07:34:13 AM	rwxr-xr-x

Editing: etl.pl

#!/usr/bin/perl

$CUBOT_HOME = "$ENV{CUBOT_HOME}";
$ETL_HOME = "$CUBOT_HOME/bin/etl/";
$ETL_LOG = "$CUBOT_HOME/logs/etllogs/";

$G_DBOutDelim = '|';
$G_User=0;
$G_PerlOutDelim = '~_';
$G_Staging_Dir = "/tmp";#This will chnge in initetl.pl
## This is stable release of ROVER : 01.Oct.2007

####   ##### #     # ##### ####  
#   #  #   # #     # #     #   # 
#   #  #   # #     # #     #   # 
####   #   #  #   #  ####  ####   
# #    #   #  #   #  #     # #   
#  #   #   #   # #   #     #  #  
#   #  #####    #    ##### #   #

##################################################################################################
#$ETLODBC = '';
$ETLODBC=1;		# Remove Comments To use ODBC then set ED_DSN for Cubot database

$Queryout=4; # Set any log level(0|1|2|3|4)

#Log levels
$MIN=0; #No Logs
$ERROR=1; #Log only Errors
$QUERY=2; #Log error and query
$INFO=3;  #Log error, query and information
$MAX=4;   #Maximum ouputs

##################################################################################################

$ETL_START_TIME = time;
export_cubotconf();
export_etlconf();
#Creating hash for all command line argument eg: if cmd is wfid=1 then $harg{"wfid"} = 1

include("config.pl");
include("func.pl");
$global_var1="$ENV{global_var1}";
$CUBETL_DATA_FROM_DATE="$ENV{CUBETL_DATA_FROM_DATE}";
$CUBETL_DATA_TO_DATE="$ENV{CUBETL_DATA_TO_DATE}";
%CMD_HARGS = makeharglist(@ARGV);

#foreach $key(keys %CMD_HARGS){
#	printlog($MAX, "$key----->$CMD_HARGS{$key}");
#	}

$ETLUSER=$CMD_HARGS{'etluser'};
$ETLUSER="cron" if $ETLUSER eq "";

$Queryout=$CMD_HARGS{'Queryout'} if $CMD_HARGS{'Queryout'} ge '0';

$ENV{'ED_DB'} = $CMD_HARGS{'db'} if defined $CMD_HARGS{'db'};

#printtime

open_etllogs();

initchk();

include("initetl.pl");

#Export CUBOT.CONF as env setting
setcubot_env();

#Init ETL dir(s)
initetl();
if($CMD_HARGS{'Permission'} ne ''){
	$G_User=1;
}

#Will Check for CUBOT_HOME and Params

include("rxdb.pl");
include("rxfile.pl");
include("sql.pl");
include("readwrite.pl");
include("workflow.pl");
include("task.pl");
include("xform.pl");
include("process.pl");
include("userfunc.pl");
include("valid.pl");
include("sourcequery.pl");
#include("rxsalesforce.pl");

if(defined xlsInstalled()){
	printlog ($INFO, "XLS Module Found.");
	include("rxcsv.pl");
}
else {
	printlog($ERROR, "XLS Module(Spreadsheet::ParseExcel) not Installed");
}

if(defined xmlInstalled()){
	printlog ($INFO, "XML Module Found.");
	include("rxxml.pl");
}
else {
	printlog($ERROR, "XML Module(XML::XPath) not Installed");
}

if(defined sfInstalled()){
	printlog ($INFO, "Salesforce Module Found.");
	include("rxsalesforce.pl");
}
else {
	printlog($ERROR, "Salesforce Module(WWW::Salesforce) not Installed");
}
#Chk Command Line is valid
#validateharg();

#This Variable is Global Cubot DB Connection Identifier. Caution : Don't modify 
$conn;

#Reset Globals
printlog($MAX, "Setting Globals");

$DATEWISE_LOG = '';
setglob();

if(defined $CMD_HARGS{'noetl'}){#Exec XLS, XML 
        printlog ($MAX, "Noetl, Exec XLS, XML");
	exec_xls_xml($CMD_HARGS{'fname'}, $CMD_HARGS{'maskfmt'}, $CMD_HARGS{'dir'}, $CMD_HARGS{'ident'}, $CMD_HARGS{'noetl'}, $CMD_HARGS{'allsh'});
	exit 1;
}

#Chk for Cub params & Open log files also connect to CUBDB $conn is global connection
configetl($CMD_HARGS{"db"});

#Read logseq from errors.txt and update DB.
updatelogseq_db();

#Setting Stage Params Before etl execution
$G_Stg_Outfile = "$RXXLS_DIR/out.txt";
$G_Stg_Errfile = "$RXXLS_DIR/outerr.txt";
$G_Shell_Errfile = "$RXXLS_DIR/shellerr.txt";

printlog ($INFO, "Start Actual ETL Processing");
execute_etl();
printlog ($INFO, "done ETL Processing ");
`rm -rf "$RXXLS_DIR/$$.time"`; # Removing time time stamp
#executewf($wfid);

#Clean Files , Close all type of Connections
cleanup();

############################################################################################
sub include{
	require "$ETL_HOME/@_";
	}
	
sub initchk{
#	chkenv("CUBOT_HOME", "ED_DB", "XLSDIR"); #This Will Die On Err ..
	if($ENV{"CUBOT_HOME"} && $ARGV[0] ne ''){ #This Will Die On Err ..
		printlog($INFO, "Using ETL_HOME:$ETL_HOME");
		}
	else{
		printlog($ERROR, "Exit.. as CUBOT_HOME Not Defined in '/home/cubot/conf/cubot.conf' .");
		exit;
		}
	}
	
sub xlsInstalled{
	return eval {require Spreadsheet::ParseExcel};
	}

sub xmlInstalled{
	return eval {require XML::XPath};
	}

sub sfInstalled{
	return eval {require WWW::Salesforce};
	}

sub export_cubotconf{
	my $CUBCONF;
	open($CUBCONF, "$CUBOT_HOME/conf/cubot.conf");
	while(<$CUBCONF>){
		chomp;
		$line = $_;
		@args = split / /, $line;
		$ENV{$args[0]} = $args[1];
		}
	}
sub export_etlconf{
	my $CUBCONF;
	open($CUBCONF, "$CUBOT_HOME/etl/process/etl_vars.conf");
	while(<$CUBCONF>){
		chomp;
		$line = $_;
		@args = split / /, $line;
		$ENV{$args[0]} = $args[1];
	}
}

sub exec_xls_xml{
	my ($fname, $maskfmt, $dir, $ident, $ppid, $allsh) = @_;
	$G_Purpose = 'noetl' if not defined $G_Purpose;

if($CMD_HARGS{"fname"} =~ m/\.xls/g){
		$allsh = '' if not defined $allsh;
		$ident = 'N' if not defined $ident;
		$maskfmt = '*.*' if not defined $maskfmt;
		$ETLMode = 'XLScsvGen';
		rxwritecsv($fname, $allsh, '', '', '', '');	
		return 1;
	}
	elsif($CMD_HARGS{"fname"} =~ m/\.xml/g){
		printlog($MAX, "Process XML files");
		my $dir = $CMD_HARGS{"dir"};
		$ETLMode = 'XMLcsvGen';
		my $schmfile = write_xmlformat2csv($dir, $maskfmt);
		printlog ($INFO, "Writing XML Data 2 CSV: $G_Purpose");
		write_xmldata2csv($dir, $maskfmt);
		return 1;
	}
	else{
		printlog ($ERROR, "NOETL: Nothing Executed");
	}
	return 1;
}