#! perl

#
# Script to copy files from one place to another, using manifests
# of files to be copied.  Manifests are moved from directory to directory,
# as their listed files are copied.  This script is intended to be run
# unattended, overnight.
#
# See end of script for the manifest format.
#
# To start using it, run it once, to create the directories.
# Then, edit the originRoot and destRoot below, to match your
# paths.  Test by dragging manifests to new_jobs and running the script.
# If it works correctly, the files should copy, and the manifest moved to
# old_jobs.
#
# Then, once you are sure it's working, add it as a scheduled job
# via the control panel.  Drag some manifests into new_jobs, wait, and
# read the logs and look in old_jobs.  If the script croaks, a manifest
# will be left in running_job.  If there are errors that don't stop the
# script, they will be moved to error_jobs.
#
# After your batches are run and files are copied, and you've checked
# that the copies work, you can drag the old_jobs into delete_jobs.
# The next time the script runs, the original files will be deleted.
# 

# NOTE: the double-backslashes are correct.  
# Use them when you mean a single backslash.

# path to where the files are kept now
$originRoot = "C:\\gistest\\";
$otherOriginRoot = "G:\\";

# path to where the files are going to be kept
$destRoot = "C:\\gistdest\\";

###########################################################################
use File::Copy;
use File::Compare;
use File::Path;
use File::Basename;

# fix up the root to work in regexes
$originRootRE = $originRoot;
$originRootRE =~ s/\\/\\\\/g;
$destRootRE = $destRoot;
$destRootRE =~ s/\\/\\\\/g;

# create directories
mkpath( [ 'new_delete_jobs','error_jobs','new_jobs','old_jobs','running_job', 'old_delete_jobs', 'error_delete_jobs', 'new_manifests' ]);
mkpath( $destRoot );



open LOG, ">>runlog.txt";
lg("START *** COPY FILES IN MANIFEST - job started ".localtime()."\n");
opendir(DH,'new_jobs');
while($line = readdir(DH))
{
	next if ($line =~ /^\./);
	copyWith( $line );
}
closedir DH;

lg("DELETE FILES IN MANIFEST - job started ".localtime()."\n");
opendir(DH,'delete_jobs');
while($line = readdir(DH))
{
	next if ($line =~ /^\./);
	deleteWith( $line );
}
closedir DH;

lg("END *** job ended ".localtime()."\n\n\n");

exit;

sub copyWith($)
{
	$file = shift(@_);
	$path = 'new_jobs\\'.$file;
	lg("OPENING $path\n");
	## move the manifest file into running_job
	$job = "running_job\\$file";
	$final = "old_jobs\\$file";
	$errorjobs = "error_jobs\\$file";
	lg("JOB: moving $path to $job\n");
	rename $path, $job;

	my $errors = 0;
	open FH,"<$job";
	while( <FH> )
	{
		@filepaths = getAllFilePaths( $_ );
		/(.+?): ([A-Z]:\\.+)$/;
		$fromPath = $2;

		next if (@filepaths == 0);  ## continue if there are not paths here

		map {
			# mangle the path...
			$toPath = $fromPath;
			$toPath =~ s/$originRootRE/$destRoot/;
			lg("COPY $fromPath to $toPath\n");

			if ( $fromPath eq $toPath )
			{
				lg("ERROR: source and destination are the same.\n");
				$errors = 1;
			}

			if (! -e $fromPath )
			{
				lg("ERROR: $fromPath does not exist\n");
				$errors = 1;
			}
	
			# build the destination dir
			$dirname  = dirname($toPath);		
			eval { mkpath($dirname) };
			if ($@) {
				lg("Couldn't create $dirname: $@\n");
			}
		
			copy( $fromPath, $toPath ) or lg("ERROR: copy from $fromPath to $toPath failed\n");
			if (! -e $toPath )
			{
				lg("ERROR: $toPath does not exist\n");
				$errors = 1;
			}

			# check that the new file matches the old
			$matches = (compare( $fromPath, $toPath ) == 0);
			if ($matches)
			{
				lg("COMPARE: $toPath was copied correctly.\n");
			}
			else
			{
				lg("ERROR: $fromPath and $toPath don't match.\n");
				$errors = 1;
			}
		} @filepaths;
	}
	close FH;
	if ($errors)
	{
		lg("ERROR: there was an error with $job, so it'll be moved to the errors folder\n");
		rename $job, $errorjobs;
	}
	else
	{
		lg("JOB: moving $job to $final\n");
		rename $job, $final;
	}
}
sub deleteWith($)
{
	$file = shift(@_);
	$path = 'new_delete_jobs\\'.$file;
	lg("OPENING $path\n");
	## move the manifest file into running_job
	$job = "running_job\\$file";
	$final = "old_delete_jobs\\$file";
	$errorjobs = "error_delete_jobs\\$file";
	lg("JOB: moving $path to $job\n");
	rename $path, $job;

	$errors = 0;
	open FH,"<$job";
	while( <FH> )
	{
		$fromPath = '';
		/(.+?): ([A-Z]:\\.+)$/;
		$fromPath = $2;

		next if ($fromPath eq '');
		next if (! -e $fromPath);

		unlink( $fromPath ) or (++$errors && lg("ERROR: cannot delete $fromPath\n"));
	}
	close FH;
	if ($errors)
	{
		lg("ERROR: there was an error with $job, so it'll be moved to the errors folder\n");
		rename $job, $errorjobs;
	}
	else
	{
		lg("JOB: moving $job to $final\n");
		rename $job, $final;
	}
}

sub lg
{
	$line = shift(@_);
	print $line;
	print LOG $line;
}

sub getAllFilePaths
{
	$line = shift(@_);
	chomp $line;
	@result = ();
	if ($line =~ /^[A-Z]:\\/)    ## this line just has a file name
	{
		$p = $line;
	}
	elsif ($line =~ /^(.+):  ([A-Z]:\\.+)$/)     ## this line is a shape file
	{
		$shpname = $1;
		$path = $2;
		if ($path =~ /^.+(\\.+\.mdb\\.+)$/)  ## this line is a shape in an mdb file
		{
			$path = $1;
		}
		$p = $path.'.dbf';
		if (-e $p) { push @result, $p; }
		$p = $path.'.prj';
		if (-e $p) { push @result, $p; }
		$p = $path.'.sbn';
		if (-e $p) { push @result, $p; }
		$p = $path.'.sbn.xml';
		if (-e $p) { push @result, $p; }
		$p = $path.'.sbx';
		if (-e $p) { push @result, $p; }
		$p = $path.'.shp';
		if (-e $p) { push @result, $p; }
		$p = $path.'.shp.xml';
		if (-e $p) { push @result, $p; }
		$p = $path.'.shx';
		if (-e $p) { push @result, $p; }
	}
	else ## we ignore the rest
	{
		return (); ## return the empty set
	}
	return @result;
}

#
# The manifest format is 
# some text here: C:\path\to\file
#
# That's it.  The first colon marks the end of the label.  After that,
# it expects a capital letter, colon, and path-like string to the end 
# of the line.
#