OXIESEC PANEL

Name	Size	Modified	Perms
📁 ..	-	02/03/2023 08:06:37 AM	rwxr-xr-x
📄 config.pl	6.53 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 dataextract.sh	2.51 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 etl.pl	5.46 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 etlproc	11.81 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 func.pl	4.01 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 globvar.sh	330 bytes	02/03/2023 07:34:08 AM	rw-r--r--
📄 initetl.pl	1.88 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 libawk.awk	3.07 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 makeXtab2NormalExcel.pl	5.25 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 process.pl	6.92 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 process_changed.pl	5.69 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 readwrite.pl	4.16 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 rxcsv.pl	7.94 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 rxdb.pl	8.84 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 rxfile.pl	4.72 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 rxsalesforce.pl	3.33 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 rxsalesforce.pl.bak	3.33 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 rxxml.pl	12.66 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 rxxml2.pl	4.18 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 s.log	8.86 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 sendmail.pl	1.06 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 sourcequery.pl	4.17 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 sql.pl	13.89 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 sql_mssql.pl	13.89 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 store.pl	7.6 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 task.pl	30.55 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 task_mssql.pl	30.8 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 tmpxml.pl	1.49 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 userfunc.pl	5.5 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 valid.pl	3.9 KB	02/03/2023 07:34:08 AM	rw-r--r--
📄 workflow.pl	3.72 KB	02/03/2023 07:34:08 AM	rwxr-xr-x
📄 xform.pl	2.62 KB	02/03/2023 07:34:08 AM	rw-r--r--

Editing: rxxml.pl

#!/usr/bin/perl
#use warnings;

use XML::XPath;

my $G_XPATH_curtablelevel=0;
my $G_XPATH_lasttablelevel=0;
my $G_XPATH_fulltag = "";
my @G_XPATH_columnslist = ();
my @G_XPATH_taglist = ();
my $G_nthxmltable = 0;

my $DELIM = $G_PerlOutDelim;

sub reset_xmlvars{
	$G_XPATH_curtablelevel=0;
	$G_XPATH_lasttablelevel=0;
	$G_XPATH_fulltag = "";
	@G_XPATH_columnslist = ();
	@G_XPATH_taglist = ();
	$G_nthxmltable = 0;
}
=head
#ARGV[0] = DirName 
#ARGV[1] = MaskFormat
require "config.pl";
require "rxcsv.pl";
require "func.pl";
$RXXLS_DIR="$RXXLS_DIR/xldir/";
$inxmldir = $ARGV[0];
$indirmask = $ARGV[1];

print time, "\n";
write_xmlformat2csv($inxmldir, $indirmask);
print time, "\n";
write_xmldata2csv($inxmldir, $indirmask);
print time, "\n";
=cut

sub getConvProperMaskFormat{
	my $maskstr = shift;

$maskstr =~ s/\*/STAR/gi;
	$maskstr =~ s/\./DOT/gi;
	$maskstr =~ s/\?/QUE/gi;
	$maskstr =~ s/[^A-Z0-9]/_/gi;

return $maskstr;
}

sub write_xmlformat2csv{
	my $dirname = shift;
	my $mask = shift;

reset_xmlvars();
	$mask = "*.xml" if((not defined $mask) or ($mask eq ''));

$dirname = "$dirname/";
	$dirname =~ s/\/\//\//g;

if ( !-d $dirname ){
		printlog($ERROR, "Error: $dirname does not exist");
		return -9999 ;
	}

printlog($INFO, "Started Getting Mask files: ", time);
	my @maskedfiles = get_masked_files($dirname, $mask);
	printlog($INFO, "Aftr get_masked_files: ", time);
	my $input_file = $maskedfiles[0];
	my $dname = getdirname($dirname);

$maskstr = getConvProperMaskFormat($mask);

$dname = $dname.".".$maskstr;

$G_xml_stg_dirname = '';
	$G_xml_stg_dirname = "$RXXLS_DIR"."/XML".$dname;

mkdir($G_xml_stg_dirname);#or die "Can't Create Staging Dir: $!";

my $schmfile = "$G_xml_stg_dirname/$dname.fmt";

my @fmtstat = stat $schmfile;
	printlog($MAX, "Time Stat : File = $fmtstat[9] ******************** ETL Start=$ETL_START_TIME");

if($fmtstat[9] >= $ETL_START_TIME){
		printlog($INFO, "Format File Is Recent: Not Generating Again");
		return -9999;
	}

if(defined @fmtstat){
		printlog($INFO, "Format file already exist: Not Generating Again");
		touchfile($schmfile);
		return -9998;
	}

my $xmlfile = $input_file;
	$xmlfile =~ s/fname=//gsi;

printlog($INFO, "Generating Format file...Now Parsing File : $xmlfile");
	my $xpath = XML::XPath->new(filename => $xmlfile) or return $G_Err = "Can't Process XML File : $xmlfile";
	printlog($INFO, "Now Making Column list");
	$lastln = '';
	foreach my $row($xpath->findnodes("/")){
		foreach my $nodelist(@$row){
			if(ref($nodelist) eq "ARRAY"){
				foreach my $node(@$nodelist){
					if(ref($node) eq "XML::XPath::Node::Element"){
						$G_XPATH_curtablelevel = 0;
						$G_XPATH_lasttablelevel = 0;
						Print_XML_NodeName($node);
						$G_XPATH_curtablelevel++;
						#$G_XPATH_lasttablelevel;
						get_XPath_List($node);
					}
				}
			}
		}
	}

printlog($MAX, "Now creating format file $schmfile");
	open($XMLFMTFP, ">$schmfile") or $G_Err = $! && return "($schmfile)Can't generate fmt file(Check Permission , Disk Space ..).";

my $ix = 0;
	my $toplevelcols = "";
	printlog($MAX, "Now Printing Column listing into format file\n");
	my %tagprinted = ();
	my $tab = 0;
	my $tagnum = 1;
	foreach my $collist(@G_XPATH_columnslist){
#    printlog($MAX, "[]******* @$collist *******\n");
#    print ">>>>> ###:$toplevelcols <<<<<<<<<\n";
		$ix = 0;
		$XMLDATA = "DATA$tagnum-$G_XPATH_taglist[$tab-1]";
		$XMLFMT = "TAB$tagnum";

if(defined $tagprinted{$G_XPATH_taglist[$tab-1]}){
			printlog($MAX, "Repeated tag : $G_XPATH_taglist[$tab-1]");	
			$tab++;
			next;
		}

if($tab != 0){
			$XMLFMT = $XMLFMT.".".@$collist[0].".";#First $col will be tabname
			$XMLDATA = $XMLDATA.'[';
			if($tab != 1){
				$XMLFMT = $XMLFMT."$toplevelcols";
			}
			foreach my $col(@$collist){
				if($ix eq 0){
				}
				else{
					$XMLFMT = $XMLFMT."$col:";
					$XMLDATA = $XMLDATA."$col:";
					if($tab == 1){
						$toplevelcols = $toplevelcols."$col:";
#            print ">>>>> TTT:$toplevelcols <<<<<<<<<\n";
					}
				}
				$ix++;
			}
			chop($XMLFMT);
			chop($XMLDATA);

print $XMLFMTFP "$XMLFMT\n";
			print $XMLFMTFP "$XMLDATA]\n";
			$tagprinted{$G_XPATH_taglist[$tab-1]} = 1;
			$tagnum++;
		}
		$tab++;
	}
	close($XMLFMTFP);
	printlog($MAX, "Finished Writing XML Format File for total Tables: $G_nthxmltable");

return ($schmfile);
}

sub get_XPath_List{
	my $parnode = shift;
	my $ir = "0";

my $childlist = $parnode->getChildNodes();
	foreach my $child(@$childlist){
		if(ref($child) eq "XML::XPath::Node::Element"){
			last;
		}
	}

foreach my $child(@$childlist){
		if(ref($child) eq "XML::XPath::Node::Element"){
			print "\n";
			for($i=0; $i<$G_XPATH_curtablelevel; $i++){
				print "\t<$G_XPATH_curtablelevel>";
			}
			Print_XML_NodeName($child, $ir);
			$G_XPATH_curtablelevel++;
			get_XPath_List($child);
			$G_XPATH_curtablelevel--;
#      print "\n";
#      for($i=0; $i<$G_XPATH_curtablelevel; $i++){
#        print "\t<$G_XPATH_curtablelevel>";
#      }
		}
	}
}

sub Print_XML_NodeName{
	my $node = shift;
	my $colname;

$colname = XML::XPath::Node::Element::getName($node);

#print "\n[cur=$G_XPATH_curtablelevel][last=$G_XPATH_lasttablelevel]";
	if($G_XPATH_curtablelevel eq $G_XPATH_lasttablelevel){
		$ln = $colname;
		$G_XPATH_columnslist[$G_nthxmltable][$nthcolumn++] = $colname;
	}
	else{
		if($G_XPATH_curtablelevel gt $G_XPATH_lasttablelevel){
			#   print "XX";
			if($ln eq $lastln){
				if($G_nthxmltable eq 1){
					$ln = $G_XPATH_columnslist[$G_nthxmltable][1];
				}else{
					$ln = $G_XPATH_columnslist[$G_nthxmltable][0];
				}
			}
			$t = $ln;
			$lastln = $ln;

#print "[$G_XPATH_curtablelevel => $G_XPATH_fulltag]";
			$G_XPATH_fulltag = $G_XPATH_fulltag."/".$ln;
			$G_XPATH_taglist[$G_nthxmltable] = $G_XPATH_fulltag;
			#print "XXX" .  @G_XPATH_columnslist[$G_nthxmltable]  . "XXX";
#      print @{@$G_XPATH_columnslist[$G_nthxmltable])};
			$G_XPATH_columnslist[$G_nthxmltable+1][0] = $G_XPATH_columnslist[$G_nthxmltable][$nthcolumn-1];
			*x = $G_XPATH_columnslist[$G_nthxmltable]; 
			pop(@x);#Remove next tab name from cur list
			$nthcolumn=1;
			# $nthcolumn++; #:) can direct be = 1
			$G_nthxmltable++;
		}
		else{
			$ln = $colname;
		}

if($G_XPATH_curtablelevel lt $G_XPATH_lasttablelevel){
			@tagListTree = split( /\//, $G_XPATH_fulltag );
			for($i=0; $i<$G_XPATH_lasttablelevel-$G_XPATH_curtablelevel; $i++){ #VK Fix for Multi Level XML Fmt (Take path to the newly shifted level): Fix 982
				my $tag2rm  = $tagListTree[$#tagListTree - $i];
				$G_XPATH_fulltag =~ s/\/$tag2rm//gsi;
			}
		}

$G_XPATH_columnslist[$G_nthxmltable][$nthcolumn++] = $colname;
		$G_XPATH_lasttablelevel = $G_XPATH_curtablelevel;
	}
	print "[$G_XPATH_fulltag/<$colname>]";
	#print "[nthtab=$G_nthxmltable]-[FULL=$G_XPATH_fulltag]";
	$colname;
}

################################################## XML DATA PRINT FUNCTION ############################################

sub write_xmldata2csv{
	my $dir = shift; 
	my $mask = shift;

my %xmlschema;
	my @xmltaglist = ();

$mask = "*.xml" if((not defined $mask) or ($mask eq ''));
	$dir =~ s/\/\//\//g;
	my $stgxml = getdirname("$dir/");

my $maskstr = getConvProperMaskFormat($mask);

$stgxml = $stgxml.".".$maskstr;

printlog($MAX, "================= INSIDE IN XML DIR = $dir =========$stgxml=======");
	my $stgdir = "$RXXLS_DIR/XML$stgxml";
	@xmltaglist = &readschema($stgdir, "$stgxml.fmt", \%xmlschema);

processdir($dir, \%xmlschema, \@xmltaglist, $stgdir, $mask);
}

sub getdirlist{
	local $dir = shift;
	local $flag = shift;
	local @dirlist = ();
	local $DH;

$G_Err = undef;
	opendir($DH, $dir) or $G_Err = $!;

print( "Processing $flag == $dir\n");

return 'ERR' if defined $G_Err;

local $ctr = 0;
	foreach $name (readdir($DH)){
		next if $name =~ /^\./; #Ignoring all files statring with .
		if($flag eq "D"){#Only Directories
			$dirlist[$ctr++] = $name if -d "$dir/$name";
		}
		elsif ($flag eq "F"){#Only Files
			$dirlist[$ctr++] = $name if not -d "$dir/$name";
		}
		else{#All Files & Dirs
			$dirlist[$ctr++] = $name;
		}
	}

closedir($DH);
	return \@dirlist;
}

##### XML SCHEMA RELATED FUNCTIONs #####
sub readschema{
	my $dir = shift;
	my $fmt = shift;
	my $xmlschema = shift;
	my @child;
	my @root;

printlog($MAX, "Using dir=$dir and Mask=$fmt");
	$G_ERRMSG = undef;

#<-- Open Schema               -->#
	open(FP, "$dir/$fmt") or printlog($ERROR, "$G_ERRMSG = $!");

printlog($ERROR, "$dir/$fmt.fmt failed : $G_ERRMSG") if defined $G_ERRMSG;
	printlog($ERROR, $G_ERRMSG) and return 'ERR' if defined $G_ERRMSG;

my @tags = ();
	my @xmltaglist = ();
	my $ix = 0;
	while(my $readline = <FP>){
		my $schema = '';
		next if ($readline !~ m/^DATA/gs);
		chomp($readline);

#<-- on error set G_ERRMSG -->#
#		print "line : 140 ---- $readline\n";
		@tags = getxmltags($readline, \$schema, \$xtag);

$line = '';
		$line = $tags[$#tags]."[".$schema."]";
		#print "Sending >> $line <<\n";
		&getschema($line , \@root, \@child);

foreach my $taglist(@child){
			#	$taglist = @child[0];
			$$xmlschema{$xtag} = [ @$taglist ];
			$xmltaglist[$ix] = $xtag;
			$ix++;
		}

@root = ();     @child = ();
	}

close(FP);

return @xmltaglist;
}

sub getschema{
	my $str = shift;
	local *root = shift;
	local *child = shift;

local $c = -1;
	local $r = -1;

while(length($str)){
		if($str =~ m/^[A-Z][A-Z0-9_]*[\[|:\]]/gsi){
		}
		else{
			$G_ERRMSG = "Error : Wrong Symbol found in schema parse string\n";
			printlog($ERROR, "$G_ERRMSG");
			printlog($ERROR, "SCHEMA [ $str ] You may have table with no columns!! Like root");
			return 'ERR';
		}
		$str = $'; my $l = $&;
		$l =~ s/[\[|:|\]]//;

if($& eq "["){
			$root[++$r]=$l;
			$c = -1;
		}
		else{
			$child[$r][++$c] = $l;
		}
	}
}

sub getxmltags {
	my $fmtline = shift;
	my $schema = shift;
	my $xtag = shift;
	my ($data, $tagline, $sch) = $fmtline =~ m/DATA([0-9]+)\-([^\[]+)\[([^\]]+)\]/gsi;

$$schema = $sch;
	$$xtag = $tagline;

my @xmltags = ();

my $i = 0;
	my $st = '';
	while($tagline ne ""){
		my ($stag, $remtag) = $tagline =~ m/\/([^\/]+)(.*)/;
		$st = $st."/".$stag;
		$xmltags[$i] = $stag;
		$i++;
		$tagline = $remtag;
	}

return @xmltags;
}

sub processdir{
	my $xmldir = shift;
	my $xmlschema = shift;
	my $taglist = shift;
	my $stgdir = shift;
	my $mask = shift;

printlog($INFO, "Recvd Dir=$xmldir, Mask=$mask");
	#local *xmldirfiles = &getdirlist($xmldir, "F");
	my @xmldirfiles = get_masked_files($xmldir, $mask);
	my $totalfiles = $#xmldirfiles + 1;
	my $nf = 4;

if(defined $G_Purpose and ($G_Purpose eq "xformpreview")){
		printlog($MAX, "1-This is Preview Mode: $totalfiles");
		my $isPreview = $totalfiles>$nf;
		if($isPreview && $isPreview == 1){
			printlog($MAX, "2-This is Preview Mode: $#xmldirfiles");
			my @x = @xmldirfiles[1..$nf+1];
			@xmldirfiles = ();
			@xmldirfiles = @x[1..$nf];
		}
	}
	my %xmldatafp = ();
	foreach my $root(@$taglist){
		my $datafname = ">$stgdir/".getdirname("$root/");
		open($xmldatafp{$root}, $datafname);
		printlog($MAX, "OpenFileHandle  $root =open($xmldatafp{$root}, $datafname)\n");
	}

my $nthfile = 1;
	foreach my $file(@xmldirfiles){
		printlog($MAX, "[$nthfile/$totalfiles]- Currently Processing file $file");
		&processfile("$file", \%$xmlschema, \@$taglist, \%xmldatafp);
		$nthfile++;
	}

foreach my $root(@$taglist){
		close($xmldatafp{$root});
	}
}

sub processfile{
	my $file = shift;
	my $xmlschema = shift;
	my $taglist = shift;
	my $xmldatafp = shift;

$G_fname = getfname($file); # Issue 1024	
	my $xpath = XML::XPath->new(filename => $file) or return $G_ERRMSG = "Can't Process XML File : $file";
#	return 'ERR' if defined $G_ERRMSG;

foreach my $root(@$taglist){
		my $FP = $$xmldatafp{$root};
		my @partags = ();
		@partags = splitxmltag($root);
		my $child = $$xmlschema{$root};
		my $ch = 0;
		my @data;	
		my $ptag = $partags[0];
		#print "=================>>>> $root <<<<=======================\n";
		foreach $row($xpath->findnodes($root)){
			@data = ();
			$ch = 0;
			foreach my $tag(@$child){
				$data[$ch] = $row->find($tag)->string_value;
				$ch++;
			}

$dt = join $DELIM, @data;
			$xmldata{$root} = $dt;
			if((defined $partags[1]) && (defined $$xmlschema{$ptag})){#Is Child Tag?
				$OutData = $xmldata{$ptag}.$DELIM.$dt;
			}
			else{#Parent Tag
				$OutData = $dt;
			}

#print $OutData, "\n";
			print $FP $OutData, "\n";
		}
		print "OVER[$FP]=================>>>> $file <<<<=======================\n";
	}
}

sub print_xml_data_to_file{
	my $FP = shift;
	my $mydata = shift;
	my $pardata = shift;
	my $parent = shift;
	my $OutData = '';

if(defined $parent){#Is Child Tag
		$OutData = $pardata.$DELIM.$mydata;
	}
	else{#Parent Tag
		$OutData = $mydata;
	}

print $FP $OutData, "\n";
}

sub splitxmltag{
	my $tag = shift;
	my @bt = ();

my $st = '';
	while($tag ne ""){
		my ($stag, $etag) = $tag =~ m/\/([^\/]+)(.*)/;
		$st = $st."/".$stag;
		$tag = $etag;
		push(@bt, $st);
	}
	return @bt;
}

sub touchfile{
	my $file = shift;

$now = time;
	utime $now, $now, $file;
}