################################################
#                                              #
#   SYSGEM Enterprise Manager 2.3 build 5665   #
#   Copyright (C) 2009 SYSGEM AG               #
#                                              #
#   Contents   : Tasks                         #
#                                              #
#   Created by : SAcM                          #
#   Timestamp  : 2013-08-27 12:46:43           #
#                                              #
################################################
#
100 Remove Duplicates From Raw Input
102 IBM
#
101 This task script is for pre-processing of the raw input (from IBM's spreadsheet) giving references to user accounts on VMS Servers, whose comment field is to be updated.
101 
101 The Raw Input Records are expected to be structured in the following way:
101 
101 |Server|Username|Owner|Comment|
101 
101 e.g.:
101 
101 |DEV|BZ9|897/C///TRUMAN, HARRY|C991234|
101 
101 (one line per user account)
101 
101 Since the servers are clustered and account records can be from any server, then there is the possibilitiy for SYSUAF update records to be duplicated.
101 
101 This script removes duplicates and validates the input prior to it being used with the Sysgem Web Access Request system to perform batch updates of the VMS accounts.
101 
101 The Cluster Definition file has one line for each cluster, each line being a comma separated string of servers. The first server in each cluster will be the only one from a cluster that is used in the "Unique" output file. 
101 
101 Any accounts that are present in the raw input file that have further records for the same cluster will result in a single entry in the 'unique' output file and the duplicates written to the 'duplicate' output file.
101 
101 Records in the 'Anomalies' output file are when one account from a cluster, and another entry from the same cluster for the same account, but with different content of the Owner Field or Comment field, is encountered. 
101 
101 Any records containing a reference to a server that is not included in the Cluster Definitions file will be rejected and placed in the 'Unrecognised Server' output file.
101 
101 Created 2013-08-23 by Mike Schofield, Sysgem AG, Switzerland.
#
#
124 /*****************************/
124 /*                           */
124 /* CL Program                */
124 /*                           */
124 /*****************************/
124 
#
#
120 #++++++++++++++++++++++++++++++++++
120 #                                 |
120 #   Windows NT script (PERL).     |
120 #                                 |
120 #++++++++++++++++++++++++++++++++++
120 #
120 #
120 use Sysgem;
120 use strict;
120 
120 #
120 #   global variables
120 #
120 my $RawCount         = 0;
120 my $ClusterCount     = 0;
120 my $ServerCount      = 0;
120 my $UniqueRecsCount  = 0;
120 my $DupRecsCount     = 0;
120 my $AnomaliesCount   = 0;
120 my $UnknownSvrsCount = 0;
120 
120 my %PrimaryServers = ();
120 
120 my %UniqueRecords  = ();
120 
120 my @UniqueRecs     = ();
120 my @DupRecs        = ();
120 my @Anomalies      = ();
120 my @UnknownSvrs    = ();
120 
120 #
120 #  do the output files have .txt file extensions?
120 #
120 if ($INPUT_UR !~ m/.txt/){$INPUT_UR = $INPUT_UR . ".txt";}
120 if ($INPUT_DR !~ m/.txt/){$INPUT_DR = $INPUT_DR . ".txt";}
120 if ($INPUT_AN !~ m/.txt/){$INPUT_AN = $INPUT_AN . ".txt";}
120 if ($INPUT_US !~ m/.txt/){$INPUT_US = $INPUT_US . ".txt";}
120 
120 #
120 #  open the input files
120 #
120 open(IN, "< $INPUT_IN") || die "cannot open [$INPUT_IN]: $!\n"; 
120 open(CD, "< $INPUT_CD") || die "cannot open [$INPUT_CD]: $!\n"; 
120 
120 #
120 #  load the content of the input files
120 #
120 my @Cluster_Definitions = <CD>;
120 my @Raw_Input           = <IN>;
120 
120 #
120 #  close the input files
120 #
120 close(CD);
120 close(IN);
120 
120 #
120 # foreach server, define the primary server in the cluster that will be used for the updates.
120 # 
120 DefinePrimaryServers();
120 
120 #
120 #  open the output files
120 #
120 open(UR, "> $INPUT_UR") || die "cannot open [$INPUT_UR]: $!\n"; 
120 open(DR, "> $INPUT_DR") || die "cannot open [$INPUT_DR]: $!\n"; 
120 open(AN, "> $INPUT_AN") || die "cannot open [$INPUT_AN]: $!\n"; 
120 open(US, "> $INPUT_US") || die "cannot open [$INPUT_US]: $!\n"; 
120 
120 #
120 #  do we need to write the headers to the output files?
120 #
120 if ($INPUT_IH == 1)
120 {
120     WriteHeaders();
120 }
120 
120 #
120 #  Process the raw data, separating each record into the appropriate output file
120 #
120 ProcessRawData();
120 
120 #
120 #  write the output files
120 #
120 foreach my $line (@UniqueRecs)
120 {
120     print UR $line . "\n";
120 }
120 foreach my $line (@DupRecs)
120 {
120     print DR $line . "\n";
120 }
120 foreach my $line (@Anomalies)
120 {
120     print AN $line . "\n";
120 }
120 foreach my $line (@UnknownSvrs)
120 {
120     print US $line . "\n";
120 }
120 
120 #
120 #  close the output files
120 #
120 close(UR);
120 close(DR);
120 close(AN);
120 close(US);
120 
120 #
120 #  print the summaries
120 #
120 
120 print "\n";
120 print "Cluster Definitions:\n";
120 print "  Number of Clusters defined ....: $ClusterCount\n";
120 print "  Number of Servers defined .....: $ServerCount\n";
120 
120 print "\n";
120 print "Summaries:\n";
120 print "  Total Number of records found in 'Raw Input' file ...:  $RawCount\n";
120 print "  Grand Total Number of output records written ........:  " . ($UniqueRecsCount + $DupRecsCount + $AnomaliesCount + $UnknownSvrsCount) . "\n";
120 
120 print "\n";
120 print "Breakdown:\n";
120 print "  Total Number of unique records written ..............:  $UniqueRecsCount\n";
120 print "  Total Number of duplicates found ....................:  $DupRecsCount\n";
120 print "  Total Number of anomalies found .....................:  $AnomaliesCount\n";
120 print "  Total Number of records with unknown servers ........:  $UnknownSvrsCount\n";
120 
120 
120 #__________________
120 #
120 #   Subroutines
120 #__________________
120 #
120 
120 #
120 #  create a hash of servers with each server pointing to its primary server 
120 #  (the "Primary" server is defined as the first in the list for each cluster!)
120 #
120 sub DefinePrimaryServers
120 {
120     foreach my $cluster (@Cluster_Definitions)
120     {
120         $cluster = Sysgem::TrimLeftRight($cluster);
120 
120         #
120         #  ignore comment and blank lines
120         #
120         if ($cluster eq ""){next;}
120         if (substr($cluster, 0, 1) eq "#"){next;}
120 
120         my @svrs = split(",", $cluster);
120 
120         my $PrimSvr = $svrs[0];
120 
120         foreach my $svr (@svrs)
120         {
120             $svr = Sysgem::TrimLeftRight($svr);
120  
120             if (!exists($PrimaryServers{$svr}))
120             {
120                 $PrimaryServers{$svr} = $PrimSvr;
120                 $ServerCount++;
120             }
120             else
120             {
120                 print "** Error - duplicate server found in cluster Definitions file [$svr]\n";
120                 print "** Error - Process has ended prematurely.\n";
120                 die;
120             }
120         }
120 
120         $ClusterCount++;
120     }
120 
120     return;
120 }
120 
120 
120 #__________________
120 #
120 
120 #
120 #  write headers into the output files
120 #
120 sub WriteHeaders
120 {
120     #
120     #  write the headers in the 'unique records' file
120     #
120     print UR "#\n";
120     print UR "#  1. This is the parameter file for updating the comment field in VMS accounts via the \n";
120     print UR "#      Sysgem \"Web Access Request\" (WAR) DB.\n";
120     print UR "#\n";
120     print UR "#  2. First, the WAR DB is updated via the Sysgem \"Submit WAR Records\" display.\n";
120     print UR "#      The \"Submit WAR Records\" display uses this parameter as input.\n";
120     print UR "#\n";
120     print UR "#  3. Records that have been 'submitted' and waiting to be processed can be listed using the \n";
120     print UR "#      Sysgem \"List Outstanding Requests\" display.\n";
120     print UR "#  \n";
120     print UR "#  4. Submitted records are processed by the \"Process Requests\" display.\n"; 
120     print UR "#     The \"Process Requests\" display can be run interactively or in batch mode.\n";
120     print UR "#\n";
120     print UR "#  5. The \"View Results\" display is used to display the history showing the processing \n";
120     print UR "#      status of each account update submitted.\n";
120     print UR "#\n";
120     print UR "#  6. SAcM Reports are used to record the list of users and the current content of the comment field. \n";
120     print UR "#     The Reports should be run on all servers (recording the data in a database) before the update\n";
120     print UR "#      exercise starts and after it has completed... for reconciliation purposes.\n";
120     print UR "#\n";
120     print UR "\n";
120 
120     print UR "Command=VerifyUpdateVMSAccountComment\n";
120     print UR "\n";
120 
120     print UR "|TargetSystem|Username|Owner|Comment|\n";
120     print UR "\n";
120 
120     #
120     #  write the headers in the 'duplicate records' file
120     #
120     print DR "#\n";
120     print DR "#  This is the 'Duplicates' file for storing those input records that already have \n";
120     print DR "#  an entry in the 'Unique Records' file for updating on the appropriate cluster.\n";
120     print DR "#\n";
120     print DR "\n";
120 
120     #
120     #  write the headers in the 'anomalies' file
120     #
120     print AN "#\n";
120     print AN "#  This is the anomalies file that stores those records that have failed the validation tests\n";
120     print AN "#  for some reason. The reason for each can be shown in this file by setting the checkbox:\n";
120     print AN "#\n";
120     print AN "#       \"Show Reason in Anomalies File\" \n";
120     print AN "#\n";
120     print AN "#  ...when starting the task script: \n";
120     print AN "#\n";
120     print AN "#       \"Remove Duplicates from Raw Input\"\n";
120     print AN "#\n";
120     print AN "\n";
120 
120     #
120     #  write the headers in the 'unidentified servers' file
120     #
120     print US "#\n";
120     print US "#  The following records in the Raw Input File have identified servers that are not defined in the 'Cluster Definitions' file\n";
120     print US "#\n";
120     print US "\n";
120 
120     return;
120 }
120 
120 
120 #__________________
120 #
120 
120 #
120 #  create a hash of servers with each server pointing to its primary server 
120 #  (the "Primary" server is defined as the first in the list for each cluster!)
120 #
120 sub ProcessRawData
120 {
120     foreach my $rawRecord (@Raw_Input)
120     {
120         $rawRecord = Sysgem::TrimLeftRight($rawRecord);
120 
120         #
120         #  ignore comment and blank lines
120         #
120         if ($rawRecord eq ""){next;}
120         if (substr($rawRecord, 0, 1) eq "#"){next;}
120 
120         $RawCount++;
120 
120         my @fields = split(/\|/, $rawRecord);
120 
120         my $svr = $fields[1];
120         $svr = Sysgem::TrimLeftRight($svr);
120 
120         my $account = $fields[2];
120         $account = Sysgem::TrimLeftRight($account);
120 
120         my $owner = $fields[3];
120         $owner = Sysgem::TrimLeftRight($owner);
120 
120         my $comment = $fields[4];
120         $comment = Sysgem::TrimLeftRight($comment);
120 
120         #
120         #  validate
120         #
120         if ($svr eq "" || $account eq "" || $owner eq "")
120         {
120             #
120             #  server, account, owner fields are mandatory
120             #
120             my $reason = "";
120 
120             if ($INPUT_IR == 1){$reason = "  \t\t<<<< server, account, and owner fields are mandatory";}
120 
120             push(@Anomalies, $rawRecord . $reason);
120 
120             $AnomaliesCount++;
120 
120             next;
120         }
120 
120         #
120         #  unknown server?
120         #
120         if (!exists($PrimaryServers{$svr})) 
120         {
120             push(@UnknownSvrs, $rawRecord);
120 
120             $UnknownSvrsCount++;
120 
120             next;
120         }
120         
120 
120         #
120         #  duplicate?
120         #
120         my $primSvr = $PrimaryServers{$svr};
120 
120 
120         my $uniqueKey = $primSvr . "\|" . $account;
120 
120         if (!exists($UniqueRecords{$uniqueKey}))
120         {
120             #
120             #  first time we have come across this account in this cluster,
120             #  record it as a unique record for output
120             #
120             $UniqueRecords{$uniqueKey} = $owner . "\|" . $comment;
120 
120             push(@UniqueRecs, "\|" . $uniqueKey . "\|" . $owner . "\|" . $comment . "\|");
120 
120             $UniqueRecsCount++;
120 
120             next;
120         }
120         else
120         {
120             #
120             #  we have see it before - is it an Anomaly?
120             #
120             if ($UniqueRecords{$uniqueKey} eq $owner . "\|" . $comment)
120             {
120                 #
120                 #  not an anomaly - just a duplicate
120                 #
120                 push(@DupRecs, "\|" . $svr . "\|" . $account . "\|" . $owner . "\|" . $comment . "\|");
120 
120                 $DupRecsCount++;
120 
120                 next;
120             }
120             else
120             {
120                 #
120                 #  it is an anomaly - seen the cluster account before, but the owner and / or comment fields differ to previous values
120                 #
120                 my $reason = "";
120 
120                 if ($INPUT_IR == 1){$reason = " \t\t<<<< Values differ to the values already accepted in the 'unique' file: [\|" . $uniqueKey . "\|" . $UniqueRecords{$uniqueKey} . "\|]";}
120 
120                 push(@Anomalies, "\|" . $svr . "\|" . $account . "\|" . $owner . "\|" . $comment . "\|" . $reason);
120 
120                 $AnomaliesCount++;
120 
120                 next;
120             }
120         }
120     }
120 
120     return;
120 }
#
#
121 #!/bin/ksh -ph
121 #
121 #{{SHELL_IRIX  #!/usr/bin/ksh
121 #{{SHELL_LINUX #!/bin/sh
121 #
121 #+++++++++++++++++++++++++++
121 #                          |
121 #   UNIX Shell script.     |
121 #                          |
121 #+++++++++++++++++++++++++++
121 #
121 #
121 #
#
122 $!+++++++++++++++++++++++++++++
122 $!                            |
122 $!  VMS command file (DCL).   |
122 $!                            |
122 $!+++++++++++++++++++++++++++++
122 $!
122 $!
#
140 This task script is for pre-processing of the raw input (from IBM's spreadsheet) giving references to user accounts on VMS Servers, whose comment field is to be updated.
140 
140 The Raw Input Records are expected to be structured in the following way:
140 
140 |Server|Username|Owner|Comment|
140 
140 e.g.:
140 
140 |DEV|BZ9|897/C///TRUMAN, HARRY|C991234|
140 
140 (one line per user account)
140 
140 Since the servers are clustered and account records can be from any server, then there is the possibilitiy for SYSUAF update records to be duplicated.
140 
140 This script removes duplicates and validates the input prior to it being used with the Sysgem Web Access Request system to perform batch updates of the VMS accounts.
140 
140 The Cluster Definition file has one line for each cluster, each line being a comma separated string of servers. The first server in each cluster will be the only one from a cluster that is used in the "Unique" output file. 
140 
140 Any accounts that are present in the raw input file that have further records for the same cluster will result in a single entry in the 'unique' output file and the duplicates written to the 'duplicate' output file.
140 
140 Records in the 'Anomalies' output file are when one account from a cluster, and another entry from the same cluster for the same account, but with different content of the Owner Field or Comment field, is encountered. 
140 
140 Any records containing a reference to a server that is not included in the Cluster Definitions file will be rejected and placed in the 'Unrecognised Server' output file.
140 
140 Created 2013-08-23 by Mike Schofield, Sysgem AG, Switzerland.
145 
144 
141 
146 
142 
143 
130 XX
131 ~f~oRemove Duplicate Account Entries (Resulting From Clustered Servers)
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XY
131 
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XA
131 ~bInput:
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 1
157 1
&
#
130 CD
131    Cluster Definitions
132 
133 14
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 IN
131    Raw Input Text File
132 
133 14
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XZ
131 ~BOutput:
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 UR
131    1. Unique Records
132 
133 22
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 DR
131    2. Duplicate Records
132 
133 22
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 AN
131    3. Anomalies
132 
133 22
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 US
131    4. Unrecognised Servers
132 
133 22
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XB
131 ~bOutput Format:
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 1
157 1
&
#
130 IR
131 
132 Include Reasons in 'Anomalies' File?
133 1
134 
135 0
136 
137 
138 16777215
139 1
150 
151 000  
152 
153 0
154 0
156 1
157 1
&
#
130 IH
131 
132 Include Headers in Output Files?
133 1
134 
135 0
136 
137 
138 16777215
139 1
150 
151 000  
152 
153 0
154 0
156 1
157 1
&
#
130 XC
131 
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
@
#
100 Add leading and trailing vertical bar
102 IBM
#
#
#
124 /*****************************/
124 /*                           */
124 /* CL Program                */
124 /*                           */
124 /*****************************/
124 
#
#
120 #++++++++++++++++++++++++++++++++++
120 #                                 |
120 #   Windows NT script (PERL).     |
120 #                                 |
120 #++++++++++++++++++++++++++++++++++
120 #
120 #
120 use Sysgem;
120 use strict;
120 
120 #
120 #   global variables
120 #
120 my $InCount  = 0;
120 my $OutCount = 0;
120 
120 my @InRecs     = ();
120 my @OutRecs    = ();
120 
120 #
120 #  does the output file have .txt file extension?
120 #
120 if ($INPUT_OU !~ m/.txt/){$INPUT_OU = $INPUT_OU . ".txt";}
120 
120 #
120 #  open the input file
120 #
120 open(IN, "< $INPUT_IN") || die "cannot open [$INPUT_IN]: $!\n"; 
120 
120 #
120 #  load the content of the input file
120 #
120 my @InRecs = <IN>;
120 
120 #
120 #  close the input file
120 #
120 close(IN);
120 
120 #
120 #  open the output file
120 #
120 open(OU, "> $INPUT_OU") || die "cannot open [$INPUT_OU]: $!\n"; 
120 
120 #
120 #  write the output files
120 #
120 foreach my $line (@InRecs)
120 {
120     chomp $line;
120     print OU "\|" . $line . "\|\n";
120 }
120 
120 #
120 #  close the output files
120 #
120 close(OU);
120 
120 #
120 #  print the summaries
120 #
120 my $count = @InRecs;
120 
120 print "\n";
120 print "Summary:\n";
120 print "  Number of lines read from input file and written to output file ....: $count\n";
120 
#
#
121 #!/bin/ksh -ph
121 #
121 #{{SHELL_IRIX  #!/usr/bin/ksh
121 #{{SHELL_LINUX #!/bin/sh
121 #
121 #+++++++++++++++++++++++++++
121 #                          |
121 #   UNIX Shell script.     |
121 #                          |
121 #+++++++++++++++++++++++++++
121 #
121 #
121 #
#
122 $!+++++++++++++++++++++++++++++
122 $!                            |
122 $!  VMS command file (DCL).   |
122 $!                            |
122 $!+++++++++++++++++++++++++++++
122 $!
122 $!
#
145 
144 
141 
146 
142 
143 
130 XX
131 ~f~oAdd a leading and trailing vertical bar to each line of the input file
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XY
131 
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XA
131 ~bInput:
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 1
157 1
&
#
130 IN
131    Input Text File
132 
133 14
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XZ
131 ~BOutput:
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 OU
131    Output Text File
132 
133 22
134 
135 1
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
130 XC
131 
132 
133 4
134 
135 0
136 
137 
138 16777215
139 
150 
151 000  
152 
153 0
154 0
156 3
157 1
&
#
@
#
#
#   License Information.
#
#{{PRODUCT:Tasks
#{{CREATED_BY:Sysgem AG,Zurich,8008,Switzerland
#
#   Checksum.
#
#{{CHECKSUM:C56E-38B0-65A0-903C-6A64-4F48-228D-08C1