*** /home/knight/cons/multi/cons Tue Apr 29 16:11:28 1997 --- cons.multi Sun Feb 6 20:43:08 2000 *************** *** 1,4 **** ! #!/usr/local/bin/perl # Cons: A Software Construction Tool. # Bob Sidebotham (rns@fore.com), FORE Systems, 1996. --- 1,7 ---- ! #!/usr/swlocal/bin/perl ! ! # TODO: make multi stuff work correctly with new jobs ! # make -pa work correctly. Right now it won't work at all. # Cons: A Software Construction Tool. # Bob Sidebotham (rns@fore.com), FORE Systems, 1996. *************** *** 25,30 **** --- 28,35 ---- # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. require 5.002; + # Could optimize this out. + require 'pwd.pl'; &initpwd; # Must use &chdir, not chdir. use integer; # Flush stdout each time. *************** *** 58,66 **** -r Remove construction products associated with -v Show cons version and continue processing. ! -V Show cons version and exit. -x Show this message and exit. ); # Simplify program name, if it is a path. --- 63,74 ---- -r Remove construction products associated with -v Show cons version and continue processing. ! -V Show cons version nd exit. -x Show this message and exit. + -j Do things in parallel by forking, all on this host. + -z Parallel build on multiple hosts (if you have .rhosts) + ); # Simplify program name, if it is a path. *************** *** 72,77 **** --- 80,86 ---- $param::build = 1; # Build targets $param::show = 1; # Show building of targets. $param::sigpro = 'md5'; # Signature protocol. + $param::jobclass='job::serial';# Which package to use for job handling, by default. # Display a command while executing or otherwise. This # should be called by command builder action methods. *************** *** 138,143 **** --- 147,158 ---- } elsif ($_ eq 'x') { print($usage); exit 0; + } elsif ($_ eq 'j') { + $param::maxjobs = shift(@ARGV); + $param::jobclass = 'job::async'; + die("$0: -j option requires an argument specifying the maximum number of jobs in parallel.\n") if !$param::maxjobs; + } elsif ($_ eq 'z') { + $param::jobclass = 'job::rmx'; } else { die qq($0: unrecognized option "-$_". Use -x for a usage message.\n) if $_; } *************** *** 176,182 **** # If the supplied top-level Conscript file is not in the # current directory, then change to that directory. if ($param::topfile =~ s#(.*)/##) { ! chdir($1) || die("$0: couldn't change to directory $1 ($!)\n"); } # Now handle override file. --- 191,197 ---- # If the supplied top-level Conscript file is not in the # current directory, then change to that directory. if ($param::topfile =~ s#(.*)/##) { ! &chdir($1) || die("$0: couldn't change to directory $1 ($!)\n"); } # Now handle override file. *************** *** 220,272 **** # Build the supplied target patterns. for $tgt (map($dir::top->lookup($_), @targets)) { ! my($status) = buildtarget($tgt); ! if ($status ne 'built') { ! my($path) = $tgt->path; ! if ($status eq "errors") { ! print qq($0: "$path" not remade because of errors.\n); ! $errors++; ! } elsif ($status eq "handled") { ! print qq($0: "$path" is up-to-date.\n); ! } elsif ($status eq "unknown") { ! # cons error already reported. ! $errors++; ! } elsif ($status eq "none") { ! #??? ! } else { ! print qq($0: don't know how to construct "$path".\n"); #' ! $errors++; } ! } ! } ! ! exit 0 + ($errors != 0); ! ! # Build the supplied target directory or files. Return aggregated status. ! sub buildtarget { ! my($tgt) = @_; ! if (ref($tgt) eq "dir") { ! my($result) = "none"; ! my($priority) = $priority{$result}; ! if (exists $tgt->{member}) { ! my($members) = $tgt->{member}; ! for $entry (sort keys %$members) { ! next if $entry =~ /^\./; # ignore hidden files ! my($tgt) = $members->{$entry}; ! next if ref($tgt) eq "file" && !exists($tgt->{builder}); ! my($stat) = buildtarget($members->{$entry}); ! my($pri) = $priority{$stat}; ! if ($pri > $priority) { ! $priority = $pri; ! $result = $stat; ! } } } ! return $result; ! } ! if ($param::build) { ! return build $tgt; } elsif ($param::depends) { my($path) = $tgt->path; if ($tgt->{builder}) { my(@dep) = (@{$tgt->{dep}}, @{$tgt->{sources}}); --- 235,274 ---- # Build the supplied target patterns. for $tgt (map($dir::top->lookup($_), @targets)) { ! my(@tgts); ! enumerate(\@tgts, $tgt); ! my($status); ! if ($param::build) { ! for $subtgt(@tgts) { ! fstart $subtgt; } ! my($status) = 'none'; ! for $subtgt(@tgts) { ! fwait $subtgt; ! if ($subtgt->{status} ne $status) { ! $status = $subtgt->{status} ! if $priority{$subtgt->{status}} > $priority{$result}; } } ! if ($status ne 'built') { ! my($path) = $tgt->path; ! if ($status eq "errors") { ! print qq($0: "$path" not remade because of errors.\n); ! $errors++; ! } elsif ($status eq "handled") { ! print qq($0: "$path" is up-to-date.\n); ! } elsif ($status eq "unknown") { ! # cons error already reported. ! $errors++; ! } elsif ($status eq "none") { ! #??? ! } else { ! print qq($0: don't know how to construct "$path".\n); ! $errors++; ! } ! } } elsif ($param::depends) { + # all this is wrong right now. my($path) = $tgt->path; if ($tgt->{builder}) { my(@dep) = (@{$tgt->{dep}}, @{$tgt->{sources}}); *************** *** 299,306 **** } } } ! return "none"; } # Support for "building" scripts, importing and exporting variables. --- 301,326 ---- } } } + } + + exit 0 + ($errors != 0); ! # Recursively enumerate the supplied directory or file, and return all ! # the buildable targets contained within. ! sub enumerate { ! my($list, $tgt) = @_; ! if (ref($tgt) eq "file" && exists($tgt->{builder})) { ! push(@$list, $tgt); ! } elsif (ref($tgt) eq "dir") { ! if (exists $tgt->{member}) { ! my($members) = $tgt->{member}; ! for $entry (sort keys %$members) { ! next if $entry =~ /^\./; # ignore hidden files ! my($tgt) = $members->{$entry}; ! enumerate($list, $tgt); ! } ! } ! } } # Support for "building" scripts, importing and exporting variables. *************** *** 581,587 **** # Generic builder module. Just a few default methods. Every derivable # file must have a builder object of some sort attached. Usually ! # builder objects are shared. package build; # Null signature for dynamic includes. --- 601,613 ---- # Generic builder module. Just a few default methods. Every derivable # file must have a builder object of some sort attached. Usually ! # builder objects are shared. Each subclass must provide start and ! # wait methods. Start arranges to start building the target, and ! # wait awaits completion of the build. If the action fails, the ! # status attribute of the target must be set to 'errors', otherwise ! # nothing is updated . The file package takes care of ! # making sure that only one attempt to build the target ever ! # occurs. package build; # Null signature for dynamic includes. *************** *** 593,598 **** --- 619,631 ---- # Not compatible with any other builder, by default. sub compatible { 0 } + # Combined start/wait action method + sub action { + my($self, $tgt) = @_; + $self->bstart($tgt); + $self->bwait($tgt); + } + # Builder module for the Install command. package build::install; *************** *** 606,620 **** } # Do the installation. ! sub action { my($self, $tgt) = @_; my($src) = $tgt->{sources}[0]; main::showcom("Install ${\$src->path} as ${\$tgt->path}") if $param::install; return unless $param::build; ! futil::install($src->srcpath, $tgt); ! return 1; } # Builder module for generic UNIX commands. package build::command; --- 639,658 ---- } # Do the installation. ! sub bstart { my($self, $tgt) = @_; my($src) = $tgt->{sources}[0]; main::showcom("Install ${\$src->path} as ${\$tgt->path}") if $param::install; return unless $param::build; ! if (! futil::install($src->srcpath, $tgt)) { ! $tgt->{status} = 'errors'; ! } } + # Optimized versions of these routines for install. + sub bwait {$_[0]->{status}} + sub action {&bstart} + # Builder module for generic UNIX commands. package build::command; *************** *** 636,656 **** # like gcc, etc. For this reason we don't have an includes # method. ! # Build the target using the previously specified commands. ! sub action { my($self, $tgt) = @_; - my($env) = $self->{env}; my($path) = $tgt->path; if ($param::build) { futil::mkdir($tgt->{dir}); unlink($path); # is this done already? } - # Set environment. - map(delete $ENV{$_}, keys %ENV); - %ENV = %{$env->{ENV}}; - # Handle multi-line commands. for $com (split(/\n/, $self->{com})) { my(@src) = (undef, @{$tgt->{sources}}); --- 674,691 ---- # like gcc, etc. For this reason we don't have an includes # method. ! # Arrange to start building the target ! sub bstart { my($self, $tgt) = @_; my($path) = $tgt->path; + my($job); if ($param::build) { futil::mkdir($tgt->{dir}); unlink($path); # is this done already? + $job = new job($tgt, $self->{env}->{ENV}); } # Handle multi-line commands. for $com (split(/\n/, $self->{com})) { my(@src) = (undef, @{$tgt->{sources}}); *************** *** 699,726 **** $com = join(' ', split(' ', $com)); next if $com =~ /^:/ && $com !~ /^:\S/; ! main::showcom($com); ! if ($param::build) { ! my($pid) = fork(); ! die("$0: unable to fork child process ($!)\n") if !defined $pid; ! if (!$pid) { ! # This is the child. ! exec($com); ! $com =~ s/\s.*//; ! die qq($0: failed to execute "$com" ($!). ) ! . qq(Is this an executable on path "$ENV{PATH}"?\n); ! } ! do {} until wait() == $pid; ! my($err) = $? >> 8; ! if ($err) { ! warn qq($0: *** [$path] Error $err\n); # trying to be like make. ! return undef; ! } ! } } ! # success. ! return 1; } # Return generic build script (without $<, $>, etc. bound), for --- 734,752 ---- $com = join(' ', split(' ', $com)); next if $com =~ /^:/ && $com !~ /^:\S/; ! $job->command($com) if $param::build; } ! if ($param::build) { ! $tgt->{job} = $job; ! jstart $job; ! } ! } ! ! sub bwait { ! my($self, $tgt) = @_; ! $tgt->{status} = $tgt->{job}->jwait; ! delete $self->{job}; } # Return generic build script (without $<, $>, etc. bound), for *************** *** 771,777 **** my($lib) = $dir->lookup($name); if ($lib->accessible) { last if $lib->ignore; ! if ((build $lib) eq 'errors') { $tgt->{status} = 'errors'; return undef; } --- 797,803 ---- my($lib) = $dir->lookup($name); if ($lib->accessible) { last if $lib->ignore; ! if ((build $lib $tgt) eq 'errors') { $tgt->{status} = 'errors'; return undef; } *************** *** 828,834 **** my($prog) = $dir->lookup($com); next com if $prog->ignore; if ($prog->accessible) { # XXX Not checking execute permission. ! if ((build $prog) eq 'errors') { $tgt->{status} = 'errors'; return undef; } --- 854,860 ---- my($prog) = $dir->lookup($com); next com if $prog->ignore; if ($prog->accessible) { # XXX Not checking execute permission. ! if ((build $prog $tgt) eq 'errors') { $tgt->{status} = 'errors'; return undef; } *************** *** 886,892 **** $self->{builder}->compatible(@_); } ! sub action { my($self, $invoked_tgt) = @_; return $self->{built} if exists $self->{built}; --- 912,918 ---- $self->{builder}->compatible(@_); } ! sub bstart { my($self, $invoked_tgt) = @_; return $self->{built} if exists $self->{built}; *************** *** 921,926 **** --- 947,956 ---- $self->{built}; } + sub bwait { + $_[1] + } + # Generic scanning module. package scan; *************** *** 940,946 **** push(@files, @{$inc->{$file}}); $files{$file} = sig->signature($file); } else { ! if ((build $file) eq 'errors') { $tgt->{status} = 'errors'; # tgt inherits build status return (); } --- 970,976 ---- push(@files, @{$inc->{$file}}); $files{$file} = sig->signature($file); } else { ! if ((build $file $tgt) eq 'errors') { $tgt->{status} = 'errors'; # tgt inherits build status return (); } *************** *** 1045,1050 **** --- 1075,1269 ---- $path } + # Job package. This handles the scheduling of actual construction + # activity. Different job classes exist, only one of which is active + # for any given cons invocation. The job class is also reponsible + # for handling all job-related output. Note that only the serial + # class allows handling of stdin: if you have an application such + # as metaconfig that requires interaction, then this must be invoked + # with the serial class. + package job; + + sub new { + my($class, $tgt, $execenv) = @_; + bless {tgt => $tgt, ENV => $execenv}, $param::jobclass; + } + + # Add a command to a job. All commands will be executed + # sequentially. If any command fails, the rest must be + # ignored. + sub command { + my($self, $com) = @_; + push(@{$self->{commands}}, $com); + } + + # Serial jobs: all commands are executed serially, by this process. + package job::serial; + + BEGIN { @ISA = qw(job) } + + sub jstart { + my($job) = @_; + my($tgt) = $job->{tgt}; + + # Set environment. + map(delete $ENV{$_}, keys %ENV); + %ENV = %{$job->{ENV}}; + + for $com (@{$job->{commands}}) { + main::showcom($com); + if ($param::build) { + my($pid) = fork(); + die("$0: unable to fork child process ($!)\n") if !defined $pid; + if (!$pid) { + # This is the child. + exec($com); + $com =~ s/\s.*//; + die qq($0: failed to execute "$com" ($!). ) + . qq(Is this an executable on path "$ENV{PATH}"?\n); + } + do {} until wait() == $pid; + my($err) = $? >> 8; + if ($err) { + warn qq($0: *** [$path] Error $err\n); # trying to be like make. + $job->{status} = 'errors'; + } + } + } + } + + sub jwait { + $_[0]->{status} + } + + # Parallel jobs: up to n jobs forked off at once, as controlled by the -j + # parameter. + package job::async; + + BEGIN { @ISA = qw(job) } + + sub jstart { + my($job) = @_; + my($tgt) = $job->{tgt}; + my($pid); + + # Set environment. + map(delete $ENV{$_}, keys %ENV); + %ENV = %{$job->{ENV}}; + + if ($param::build) { + while ($njobs >= $param::maxjobs) { + $pid = wait(); + die("$0: wait returned an error ($!)\n") if $pid == -1; + if ($?) { + $job{$pid}->{status} = 'errors'; + } else { # XXX this is ugly. Couldn't we use one status? + $job{$pid}->{done} = 1; + } + $njobs--; + } + $pid = fork(); + die("$0: unable to fork child process ($!)\n") if !defined $pid; + } + + if ($pid) { + # This is the parent + $njobs++; + $job->{pid} = $pid; + $job{$pid} = $job; + } else { + # This is the child. + my(@commands) = @{$job->{commands}}; + while ($com = shift(@commands)) { + main::showcom($com); + if ($param::build) { + my($pid); + $pid = fork() if @commands; + if (!$pid) { + # This is the child of the child. + exec($com); + $com =~ s/\s.*//; + die qq($0: failed to execute "$com" ($!). ) + . qq(Is this an executable on path "$ENV{PATH}"?\n); + } + do {} until wait() == $pid; + my($err) = $? >> 8; + if ($err) { + warn qq($0: *** [$path] Error $err\n); # trying to be like make. + exit($err); + } + } + } + exit(0); + } + } + + sub jwait { + my($job) = @_; + if (! $job->{status} && !$job->{done}) { + my($waitpid) = $job->{pid}; + do { + $pid = wait(); + if ($?) { + $job{$pid}->{status} = 'errors'; + } else { # XXX this is ugly. Couldn't we use one status? + $job{$pid}->{done} = 1; + } + die("$0: wait returned an error ($!)\n") if $pid == -1; + $njobs--; + } until $pid == $waitpid; + } + $_[0]->{status} + } + + package job::rmx; + + BEGIN { @ISA = qw(job) } + + sub notify { + my($job, $exitstatus) = @_; + unlink($job->{script}); + open(LOG, $job->{log}); + while () { + print $_; + } + if ($exitstatus) { + my($path) = $job->{tgt}->path; + warn qq($0: *** [$path] Error $exitstatus\n); + } + } + + sub jstart { + my($job) = @_; + my($tgt) = $job->{tgt}; + my($env) = $job->{ENV}; + my($path) = $tgt->path; + my($script) = $job->{script} = "$path#sh"; + my($log) = $job->{log} = "$path#log"; + open(SCRIPT, ">$script") || die("couldn't create $script ($!)\n"); + print SCRIPT "set -e\n" . "cd $env->{PWD}\n"; + while (($key, $val) = each %$env) { + print SCRIPT "$key=$val; export $key\n"; + } + for $com (@{$job->{commands}}) { + ($echo = $com) =~ s/'/'\\''/; + print SCRIPT "echo '$echo'\n" . "($com)\n"; + } + $rmxcom = $rmx->startproc("sh $script > $log 2>&1", \notify, $job); + die("couldn't start remote job") if !$rmxcom; # XXX + $job->{rmxcom} = $rmxcom; + } + + sub jwait { + my($job) = @_; + return 'errors' if $job->{rmxcom}->wait(); + return undef; + } + + # XXX rmx shutdown processing should be initiated somewhere when + # an error is detected. + + # Directory and file handling. Files/dirs are represented by objects. # Other packages are welcome to add component-specific attributes. package dir; *************** *** 1208,1218 **** # Build the file, if necessary. sub build { ! $_[0]->{status} || &file::_build } ! sub _build { ! my($self) = @_; if (!exists $self->{builder}) { # We don't know how to build the file. This is OK, if # the file is present in the tree. --- 1427,1440 ---- # Build the file, if necessary. sub build { ! $_[0]->{status} || (&fstart, &fwait) } ! # Start build, asynchronously if possible. ! # Parent object may be passed in optionally for better diagnostics. ! sub fstart { ! my($self, $parent) = @_; ! return if $self->{started}; if (!exists $self->{builder}) { # We don't know how to build the file. This is OK, if # the file is present in the tree. *************** *** 1220,1232 **** # This may read the signature (if externally provided) # or fabricate it (either from the file's name and timestamp) # or from the file's contents. ! return $self->{status} = 'handled'; } else { my($name) = $self->path; ! print("$0: don't know how to construct \"$name\"\n"); ! exit(1) unless $param::kflag; ! return $self->{status} = 'errors'; # xxx used to be 'unknown' } } # An associated build object exists, so we know how to build --- 1442,1460 ---- # This may read the signature (if externally provided) # or fabricate it (either from the file's name and timestamp) # or from the file's contents. ! $self->{status} = 'handled'; } else { my($name) = $self->path; ! my $warn = qq($0: don't know how to construct "$name"); ! $warn .= qq( needed by "${\$parent->path}") if $parent; ! warn("$warn\n"); ! if (!$param::kflag) { ! sig::hash::END(); ! exit(1) unless $param::kflag; ##### need better shutdown for parallel build ! } ! $self->{status} = 'errors'; } + return; } # An associated build object exists, so we know how to build *************** *** 1237,1246 **** # Will contain signatures of dependents my (@dsig); # Get signatures of statically defined dependents. ! for $dep (@{$self->{dep}}, @{$self->{sources}}) { ! if ((build $dep) eq 'errors') { # Propagate dependent errors to target. $self->{status} = 'errors'; } else { --- 1465,1481 ---- # Will contain signatures of dependents my (@dsig); + my (@deps) = (@{$self->{dep}}, @{$self->{sources}}); + + # Start any asynchronous builds we can. + for $dep (@deps) { + $dep->fstart($self); + } # Get signatures of statically defined dependents. ! # XXXXX Some question here about -k vs. not -k semantics. ! for $dep (@deps) { ! if ((fwait $dep) eq 'errors') { # Propagate dependent errors to target. $self->{status} = 'errors'; } else { *************** *** 1248,1258 **** } } - return 'errors' if $self->{status} eq 'errors'; - # Add dynamic dependent signature, if appropriate. push(@dsig, $builder->includes($self)); ! return 'errors' if $self->{status} eq 'errors'; # Compute the final signature of the file. my($sig) = sig->collect(@dsig, $builder->script); --- 1483,1493 ---- } } # Add dynamic dependent signature, if appropriate. push(@dsig, $builder->includes($self)); ! ! # If dependents have errors, cannot proceed with build of target. ! return if $self->{status}; # Compute the final signature of the file. my($sig) = sig->collect(@dsig, $builder->script); *************** *** 1260,1276 **** # Then check for currency. if (! sig->current($self, $sig)) { # We have to build/derive the file. ! if (! $builder->action($self)) { die("$0: errors constructing ${\$self->path}\n") unless $param::kflag; ! return $self->{status} = 'errors'; } else { # We only ever set status to "built" if it was really built # by us (not just because it exists). ! sig->set($self, $sig); return $self->{status} = 'built'; } - } else { - return $self->{status} = 'handled'; } } --- 1495,1521 ---- # Then check for currency. if (! sig->current($self, $sig)) { # We have to build/derive the file. ! $builder->bstart($self); ! $self->{started} = $sig; ! } else { ! $self->{status} = 'handled'; ! } ! } ! ! # Wait for build already started. ! sub fwait { ! $_[0]->{status} || do { ! my($self) = @_; ! $self->{builder}->bwait($self); ! if ($self->{status}) { die("$0: errors constructing ${\$self->path}\n") unless $param::kflag; ! return 'errors'; } else { # We only ever set status to "built" if it was really built # by us (not just because it exists). ! sig->set($self, $self->{started}); return $self->{status} = 'built'; } } } *************** *** 1309,1319 **** sub install { my($sp, $tgt) = @_; my($tp) = $tgt->path; ! return if $tp eq $sp; ! return if link($sp, $tp); unlink($tp); futil::mkdir($tgt->{dir}); ! return if link($sp, $tp); futil::copy($sp, $tp); } --- 1554,1564 ---- sub install { my($sp, $tgt) = @_; my($tp) = $tgt->path; ! return 1 if $tp eq $sp; ! return 1 if link($sp, $tp); unlink($tp); futil::mkdir($tgt->{dir}); ! return 1 if link($sp, $tp); futil::copy($sp, $tp); } *************** *** 1504,1506 **** --- 1749,1755 ---- $md5->add(join('', @sigs)); unpack("H*", $md5->digest()); } + + # Support for remote build. + package remote; +