From 4a6aeb670f74ef895878631bc3d832d08e1cb321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Fri, 3 Jul 2015 00:05:16 +0200 Subject: [PATCH] derivations: Add #:substitutable?, distinguished from #:local-build?. Fixes . * guix/derivations.scm (substitutable-derivation?): Rewrite to check for "allowSubstitutes". (derivation): Add #:substitutable? parameter. [user+system-env-vars]: Honor it. (build-expression->derivation): Add #:substitutable? and honor it. * guix/gexp.scm (gexp->derivation): Likewise. * tests/derivations.scm ("derivation-prerequisites-to-build and substitutes, non-substitutable build"): Use #:substitutable? instead of #:local-build?. ("substitutable-derivation?", "derivation-prerequisites-to-build and substitutes, local build"): New tests. * guix/download.scm (url-fetch): Adjust comment. * guix/git-download.scm (git-fetch): Likewise. * guix/build-system/gnu.scm (gnu-build, gnu-cross-build): Use #:substitutable? instead of #:local-build?. * doc/guix.texi (Derivations, G-Expressions): Adjust accordingly. --- doc/guix.texi | 15 +++++++++---- guix/build-system/gnu.scm | 10 ++------- guix/derivations.scm | 29 ++++++++++++++++++-------- guix/download.scm | 4 ++-- guix/gexp.scm | 5 +++-- guix/git-download.scm | 1 + tests/derivations.scm | 44 ++++++++++++++++++++++++++++++++------- 7 files changed, 76 insertions(+), 32 deletions(-) diff --git a/doc/guix.texi b/doc/guix.texi index fdd8fdf9ae..284d667f34 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -2582,7 +2582,8 @@ a derivation is the @code{derivation} procedure: @var{args} [#:outputs '("out")] [#:hash #f] [#:hash-algo #f] @ [#:recursive? #f] [#:inputs '()] [#:env-vars '()] @ [#:system (%current-system)] [#:references-graphs #f] @ - [#:allowed-references #f] [#:leaked-env-vars #f] [#:local-build? #f] + [#:allowed-references #f] [#:leaked-env-vars #f] [#:local-build? #f] @ + [#:substitutable? #t] Build a derivation with the given arguments, and return the resulting @code{} object. @@ -2612,6 +2613,11 @@ When @var{local-build?} is true, declare that the derivation is not a good candidate for offloading and should rather be built locally (@pxref{Daemon Offload Setup}). This is the case for small derivations where the costs of data transfers would outweigh the benefits. + +When @var{substitutable?} is false, declare that substitutes of the +derivation's output should not be used (@pxref{Substitutes}). This is +useful, for instance, when building packages that capture details of the +host CPU instruction set. @end deffn @noindent @@ -2651,7 +2657,7 @@ is now deprecated in favor of the much nicer @code{gexp->derivation}. [#:outputs '("out")] [#:hash #f] [#:hash-algo #f] @ [#:recursive? #f] [#:env-vars '()] [#:modules '()] @ [#:references-graphs #f] [#:allowed-references #f] @ - [#:local-build? #f] [#:guile-for-build #f] + [#:local-build? #f] [#:substitutable? #t] [#:guile-for-build #f] Return a derivation that executes Scheme expression @var{exp} as a builder for derivation @var{name}. @var{inputs} must be a list of @code{(name drv-path sub-drv)} tuples; when @var{sub-drv} is omitted, @@ -2674,7 +2680,8 @@ terminates by passing the result of @var{exp} to @code{exit}; thus, when @code{%guile-for-build} fluid is used instead. See the @code{derivation} procedure for the meaning of -@var{references-graphs}, @var{allowed-references}, and @var{local-build?}. +@var{references-graphs}, @var{allowed-references}, @var{local-build?}, +and @var{substitutable?}. @end deffn @noindent @@ -3163,7 +3170,7 @@ information about monads.) [#:module-path @var{%load-path}] @ [#:references-graphs #f] [#:allowed-references #f] @ [#:leaked-env-vars #f] @ - [#:local-build? #f] [#:guile-for-build #f] + [#:local-build? #f] [#:substitutable? #t] [#:guile-for-build #f] Return a derivation @var{name} that runs @var{exp} (a gexp) with @var{guile-for-build} (a derivation) on @var{system}. When @var{target} is true, it is used as the cross-compilation target triplet for packages diff --git a/guix/build-system/gnu.scm b/guix/build-system/gnu.scm index 05b6e6f680..8702c6c915 100644 --- a/guix/build-system/gnu.scm +++ b/guix/build-system/gnu.scm @@ -367,10 +367,7 @@ (define guile-for-build #:inputs input-drvs #:outputs outputs #:modules imported-modules - - ;; XXX: Update when - ;; is fixed. - #:local-build? (not substitutable?) + #:substitutable? substitutable? #:allowed-references (and allowed-references @@ -513,10 +510,7 @@ (define guile-for-build #:inputs (append native-drvs target-drvs) #:outputs outputs #:modules imported-modules - - ;; XXX: Update when - ;; is fixed. - #:local-build? (not substitutable?) + #:substitutable? substitutable? #:allowed-references (and allowed-references diff --git a/guix/derivations.scm b/guix/derivations.scm index 1056caa70a..f12127f0ed 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -223,10 +223,13 @@ (define (offloadable-derivation? drv) (("preferLocalBuild" . "1") #f) (_ #t))) -(define substitutable-derivation? - ;; Return #t if the derivation can be substituted. Currently the two are - ;; synonymous, see . - offloadable-derivation?) +(define (substitutable-derivation? drv) + "Return #t if DRV can be substituted." + (match (assoc "allowSubstitutes" + (derivation-builder-environment-vars drv)) + (("allowSubstitutes" . value) + (string=? value "1")) + (_ #t))) (define (derivation-output-paths drv sub-drvs) "Return the output paths of outputs SUB-DRVS of DRV." @@ -692,7 +695,8 @@ (define* (derivation store name builder args (inputs '()) (outputs '("out")) hash hash-algo recursive? references-graphs allowed-references - leaked-env-vars local-build?) + leaked-env-vars local-build? + (substitutable? #t)) "Build a derivation with the given arguments, and return the resulting object. When HASH and HASH-ALGO are given, a fixed-output derivation is created---i.e., one whose result is known in @@ -715,7 +719,10 @@ (define* (derivation store name builder args When LOCAL-BUILD? is true, declare that the derivation is not a good candidate for offloading and should rather be built locally. This is the case for small -derivations where the costs of data transfers would outweigh the benefits." +derivations where the costs of data transfers would outweigh the benefits. + +When SUBSTITUTABLE? is false, declare that substitutes of the derivation's +output should not be used." (define (add-output-paths drv) ;; Return DRV with an actual store path for each of its output and the ;; corresponding environment variable. @@ -753,6 +760,9 @@ (define (user+system-env-vars) (let ((env-vars `(,@(if local-build? `(("preferLocalBuild" . "1")) '()) + ,@(if (not substitutable?) + `(("allowSubstitutes" . "0")) + '()) ,@(if allowed-references `(("allowedReferences" . ,(string-join allowed-references))) @@ -1173,7 +1183,7 @@ (define* (build-expression->derivation store name exp ;deprecated guile-for-build references-graphs allowed-references - local-build?) + local-build? (substitutable? #t)) "Return a derivation that executes Scheme expression EXP as a builder for derivation NAME. INPUTS must be a list of (NAME DRV-PATH SUB-DRV) tuples; when SUB-DRV is omitted, \"out\" is assumed. MODULES is a list @@ -1193,7 +1203,7 @@ (define* (build-expression->derivation store name exp ;deprecated omitted or is #f, the value of the `%guile-for-build' fluid is used instead. See the `derivation' procedure for the meaning of REFERENCES-GRAPHS, -ALLOWED-REFERENCES, and LOCAL-BUILD?." +ALLOWED-REFERENCES, LOCAL-BUILD?, and SUBSTITUTABLE?." (define guile-drv (or guile-for-build (%guile-for-build))) @@ -1319,7 +1329,8 @@ (define %build-inputs #:outputs outputs #:references-graphs references-graphs #:allowed-references allowed-references - #:local-build? local-build?))) + #:local-build? local-build? + #:substitutable? substitutable?))) ;;; diff --git a/guix/download.scm b/guix/download.scm index 3f7f7badce..9a80af7de6 100644 --- a/guix/download.scm +++ b/guix/download.scm @@ -277,8 +277,8 @@ (define builder ;; In general, offloading downloads is not a good idea. ;;#:local-build? #t ;; FIXME: The above would also disable use of - ;; substitutes, so comment it out; see - ;; . + ;; substitutes on old daemons, so comment it out; + ;; see . ))))) (define* (download-to-store store url #:optional (name (basename url)) diff --git a/guix/gexp.scm b/guix/gexp.scm index 09b51b3936..49dcc99ac3 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -320,7 +320,7 @@ (define* (gexp->derivation name exp references-graphs allowed-references leaked-env-vars - local-build?) + local-build? (substitutable? #t)) "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a derivation) on SYSTEM. When TARGET is true, it is used as the cross-compilation target triplet for packages referred to by EXP. @@ -439,7 +439,8 @@ (define (graphs-file-names graphs) #:references-graphs (and=> graphs graphs-file-names) #:allowed-references allowed #:leaked-env-vars leaked-env-vars - #:local-build? local-build?)))) + #:local-build? local-build? + #:substitutable? substitutable?)))) (define* (gexp-inputs exp #:key native?) "Return the input list for EXP. When NATIVE? is true, return only native diff --git a/guix/git-download.scm b/guix/git-download.scm index f4b48d7a6b..0f2218c13e 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -90,6 +90,7 @@ (define build (gexp->derivation (or name "git-checkout") build #:system system ;; FIXME: See . + ;; Uncomment when fixed daemons are widely deployed. ;;#:local-build? #t #:hash-algo hash-algo #:hash hash diff --git a/tests/derivations.scm b/tests/derivations.scm index df5f07d117..f66ef5cdd7 100644 --- a/tests/derivations.scm +++ b/tests/derivations.scm @@ -188,10 +188,22 @@ (define prefix-len (string-length dir)) (test-assert "offloadable-derivation?" (and (offloadable-derivation? (derivation %store "foo" %bash '())) + (offloadable-derivation? ;see + (derivation %store "foo" %bash '() + #:substitutable? #f)) (not (offloadable-derivation? (derivation %store "foo" %bash '() #:local-build? #t))))) +(test-assert "substitutable-derivation?" + (and (substitutable-derivation? (derivation %store "foo" %bash '())) + (substitutable-derivation? ;see + (derivation %store "foo" %bash '() + #:local-build? #f)) + (not (substitutable-derivation? + (derivation %store "foo" %bash '() + #:substitutable? #f))))) + (test-assert "fixed-output-derivation?" (let* ((builder (add-text-to-store %store "my-fixed-builder.sh" "echo -n hello > $out" '())) @@ -614,14 +626,11 @@ (define %coreutils (null? download*) (null? build*)))))) -(test-assert "derivation-prerequisites-to-build and substitutes, local build" +(test-assert "derivation-prerequisites-to-build and substitutes, non-substitutable build" (let* ((store (open-connection)) - (drv (build-expression->derivation store "prereq-subst-local" + (drv (build-expression->derivation store "prereq-no-subst" (random 1000) - ;; XXX: Adjust once - ;; - ;; is fixed. - #:local-build? #t)) + #:substitutable? #f)) (output (derivation->output-path drv))) ;; Make sure substitutes are usable. @@ -631,13 +640,34 @@ (define %coreutils (let-values (((build download) (derivation-prerequisites-to-build store drv))) ;; Despite being available as a substitute, DRV will be built locally - ;; due to #:local-build?. + ;; due to #:substitutable? #f. (and (null? download) (match build (((? derivation-input? input)) (string=? (derivation-input-path input) (derivation-file-name drv))))))))) +(test-assert "derivation-prerequisites-to-build and substitutes, local build" + (with-store store + (let* ((drv (build-expression->derivation store "prereq-subst-local" + (random 1000) + #:local-build? #t)) + (output (derivation->output-path drv))) + + ;; Make sure substitutes are usable. + (set-build-options store #:use-substitutes? #t) + + (with-derivation-narinfo drv + (let-values (((build download) + (derivation-prerequisites-to-build store drv))) + ;; #:local-build? is not be synonymous with #:substitutable?, so we + ;; must be able to substitute DRV's output. + ;; See . + (and (null? build) + (match download + (((? string? item)) + (string=? item (derivation->output-path drv)))))))))) + (test-assert "build-expression->derivation with expression returning #f" (let* ((builder '(begin (mkdir %output)