1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package blueprint 16 17import ( 18 "bytes" 19 "context" 20 "errors" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "os" 25 "path/filepath" 26 "reflect" 27 "runtime" 28 "runtime/pprof" 29 "sort" 30 "strings" 31 "sync" 32 "sync/atomic" 33 "text/scanner" 34 "text/template" 35 36 "github.com/google/blueprint/parser" 37 "github.com/google/blueprint/pathtools" 38 "github.com/google/blueprint/proptools" 39) 40 41var ErrBuildActionsNotReady = errors.New("build actions are not ready") 42 43const maxErrors = 10 44const MockModuleListFile = "bplist" 45 46// A Context contains all the state needed to parse a set of Blueprints files 47// and generate a Ninja file. The process of generating a Ninja file proceeds 48// through a series of four phases. Each phase corresponds with a some methods 49// on the Context object 50// 51// Phase Methods 52// ------------ ------------------------------------------- 53// 1. Registration RegisterModuleType, RegisterSingletonType 54// 55// 2. Parse ParseBlueprintsFiles, Parse 56// 57// 3. Generate ResolveDependencies, PrepareBuildActions 58// 59// 4. Write WriteBuildFile 60// 61// The registration phase prepares the context to process Blueprints files 62// containing various types of modules. The parse phase reads in one or more 63// Blueprints files and validates their contents against the module types that 64// have been registered. The generate phase then analyzes the parsed Blueprints 65// contents to create an internal representation for the build actions that must 66// be performed. This phase also performs validation of the module dependencies 67// and property values defined in the parsed Blueprints files. Finally, the 68// write phase generates the Ninja manifest text based on the generated build 69// actions. 70type Context struct { 71 context.Context 72 73 // set at instantiation 74 moduleFactories map[string]ModuleFactory 75 nameInterface NameInterface 76 moduleGroups []*moduleGroup 77 moduleInfo map[Module]*moduleInfo 78 modulesSorted []*moduleInfo 79 preSingletonInfo []*singletonInfo 80 singletonInfo []*singletonInfo 81 mutatorInfo []*mutatorInfo 82 earlyMutatorInfo []*mutatorInfo 83 variantMutatorNames []string 84 85 depsModified uint32 // positive if a mutator modified the dependencies 86 87 dependenciesReady bool // set to true on a successful ResolveDependencies 88 buildActionsReady bool // set to true on a successful PrepareBuildActions 89 90 // set by SetIgnoreUnknownModuleTypes 91 ignoreUnknownModuleTypes bool 92 93 // set by SetAllowMissingDependencies 94 allowMissingDependencies bool 95 96 // set during PrepareBuildActions 97 pkgNames map[*packageContext]string 98 liveGlobals *liveTracker 99 globalVariables map[Variable]ninjaString 100 globalPools map[Pool]*poolDef 101 globalRules map[Rule]*ruleDef 102 103 // set during PrepareBuildActions 104 ninjaBuildDir ninjaString // The builddir special Ninja variable 105 requiredNinjaMajor int // For the ninja_required_version variable 106 requiredNinjaMinor int // For the ninja_required_version variable 107 requiredNinjaMicro int // For the ninja_required_version variable 108 109 subninjas []string 110 111 // set lazily by sortedModuleGroups 112 cachedSortedModuleGroups []*moduleGroup 113 114 globs map[string]GlobPath 115 globLock sync.Mutex 116 117 srcDir string 118 fs pathtools.FileSystem 119 moduleListFile string 120} 121 122// An Error describes a problem that was encountered that is related to a 123// particular location in a Blueprints file. 124type BlueprintError struct { 125 Err error // the error that occurred 126 Pos scanner.Position // the relevant Blueprints file location 127} 128 129// A ModuleError describes a problem that was encountered that is related to a 130// particular module in a Blueprints file 131type ModuleError struct { 132 BlueprintError 133 module *moduleInfo 134} 135 136// A PropertyError describes a problem that was encountered that is related to a 137// particular property in a Blueprints file 138type PropertyError struct { 139 ModuleError 140 property string 141} 142 143func (e *BlueprintError) Error() string { 144 return fmt.Sprintf("%s: %s", e.Pos, e.Err) 145} 146 147func (e *ModuleError) Error() string { 148 return fmt.Sprintf("%s: %s: %s", e.Pos, e.module, e.Err) 149} 150 151func (e *PropertyError) Error() string { 152 return fmt.Sprintf("%s: %s: %s: %s", e.Pos, e.module, e.property, e.Err) 153} 154 155type localBuildActions struct { 156 variables []*localVariable 157 rules []*localRule 158 buildDefs []*buildDef 159} 160 161type moduleAlias struct { 162 variantName string 163 variant variationMap 164 dependencyVariant variationMap 165 target *moduleInfo 166} 167 168type moduleGroup struct { 169 name string 170 ninjaName string 171 172 modules []*moduleInfo 173 aliases []*moduleAlias 174 175 namespace Namespace 176} 177 178type moduleInfo struct { 179 // set during Parse 180 typeName string 181 factory ModuleFactory 182 relBlueprintsFile string 183 pos scanner.Position 184 propertyPos map[string]scanner.Position 185 createdBy *moduleInfo 186 187 variantName string 188 variant variationMap 189 dependencyVariant variationMap 190 191 logicModule Module 192 group *moduleGroup 193 properties []interface{} 194 195 // set during ResolveDependencies 196 missingDeps []string 197 newDirectDeps []depInfo 198 199 // set during updateDependencies 200 reverseDeps []*moduleInfo 201 forwardDeps []*moduleInfo 202 directDeps []depInfo 203 204 // used by parallelVisitAllBottomUp 205 waitingCount int 206 207 // set during each runMutator 208 splitModules []*moduleInfo 209 aliasTarget *moduleInfo 210 211 // set during PrepareBuildActions 212 actionDefs localBuildActions 213} 214 215type depInfo struct { 216 module *moduleInfo 217 tag DependencyTag 218} 219 220func (module *moduleInfo) Name() string { 221 // If this is called from a LoadHook (which is run before the module has been registered) 222 // then group will not be set and so the name is retrieved from logicModule.Name(). 223 // Usually, using that method is not safe as it does not track renames (group.name does). 224 // However, when called from LoadHook it is safe as there is no way to rename a module 225 // until after the LoadHook has run and the module has been registered. 226 if module.group != nil { 227 return module.group.name 228 } else { 229 return module.logicModule.Name() 230 } 231} 232 233func (module *moduleInfo) String() string { 234 s := fmt.Sprintf("module %q", module.Name()) 235 if module.variantName != "" { 236 s += fmt.Sprintf(" variant %q", module.variantName) 237 } 238 if module.createdBy != nil { 239 s += fmt.Sprintf(" (created by %s)", module.createdBy) 240 } 241 242 return s 243} 244 245func (module *moduleInfo) namespace() Namespace { 246 return module.group.namespace 247} 248 249// A Variation is a way that a variant of a module differs from other variants of the same module. 250// For example, two variants of the same module might have Variation{"arch","arm"} and 251// Variation{"arch","arm64"} 252type Variation struct { 253 // Mutator is the axis on which this variation applies, i.e. "arch" or "link" 254 Mutator string 255 // Variation is the name of the variation on the axis, i.e. "arm" or "arm64" for arch, or 256 // "shared" or "static" for link. 257 Variation string 258} 259 260// A variationMap stores a map of Mutator to Variation to specify a variant of a module. 261type variationMap map[string]string 262 263func (vm variationMap) clone() variationMap { 264 if vm == nil { 265 return nil 266 } 267 newVm := make(variationMap) 268 for k, v := range vm { 269 newVm[k] = v 270 } 271 272 return newVm 273} 274 275// Compare this variationMap to another one. Returns true if the every entry in this map 276// is either the same in the other map or doesn't exist in the other map. 277func (vm variationMap) subset(other variationMap) bool { 278 for k, v1 := range vm { 279 if v2, ok := other[k]; ok && v1 != v2 { 280 return false 281 } 282 } 283 return true 284} 285 286func (vm variationMap) equal(other variationMap) bool { 287 return reflect.DeepEqual(vm, other) 288} 289 290type singletonInfo struct { 291 // set during RegisterSingletonType 292 factory SingletonFactory 293 singleton Singleton 294 name string 295 296 // set during PrepareBuildActions 297 actionDefs localBuildActions 298} 299 300type mutatorInfo struct { 301 // set during RegisterMutator 302 topDownMutator TopDownMutator 303 bottomUpMutator BottomUpMutator 304 name string 305 parallel bool 306} 307 308func newContext() *Context { 309 return &Context{ 310 Context: context.Background(), 311 moduleFactories: make(map[string]ModuleFactory), 312 nameInterface: NewSimpleNameInterface(), 313 moduleInfo: make(map[Module]*moduleInfo), 314 globs: make(map[string]GlobPath), 315 fs: pathtools.OsFs, 316 ninjaBuildDir: nil, 317 requiredNinjaMajor: 1, 318 requiredNinjaMinor: 7, 319 requiredNinjaMicro: 0, 320 } 321} 322 323// NewContext creates a new Context object. The created context initially has 324// no module or singleton factories registered, so the RegisterModuleFactory and 325// RegisterSingletonFactory methods must be called before it can do anything 326// useful. 327func NewContext() *Context { 328 ctx := newContext() 329 330 ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator) 331 332 return ctx 333} 334 335// A ModuleFactory function creates a new Module object. See the 336// Context.RegisterModuleType method for details about how a registered 337// ModuleFactory is used by a Context. 338type ModuleFactory func() (m Module, propertyStructs []interface{}) 339 340// RegisterModuleType associates a module type name (which can appear in a 341// Blueprints file) with a Module factory function. When the given module type 342// name is encountered in a Blueprints file during parsing, the Module factory 343// is invoked to instantiate a new Module object to handle the build action 344// generation for the module. If a Mutator splits a module into multiple variants, 345// the factory is invoked again to create a new Module for each variant. 346// 347// The module type names given here must be unique for the context. The factory 348// function should be a named function so that its package and name can be 349// included in the generated Ninja file for debugging purposes. 350// 351// The factory function returns two values. The first is the newly created 352// Module object. The second is a slice of pointers to that Module object's 353// properties structs. Each properties struct is examined when parsing a module 354// definition of this type in a Blueprints file. Exported fields of the 355// properties structs are automatically set to the property values specified in 356// the Blueprints file. The properties struct field names determine the name of 357// the Blueprints file properties that are used - the Blueprints property name 358// matches that of the properties struct field name with the first letter 359// converted to lower-case. 360// 361// The fields of the properties struct must be either []string, a string, or 362// bool. The Context will panic if a Module gets instantiated with a properties 363// struct containing a field that is not one these supported types. 364// 365// Any properties that appear in the Blueprints files that are not built-in 366// module properties (such as "name" and "deps") and do not have a corresponding 367// field in the returned module properties struct result in an error during the 368// Context's parse phase. 369// 370// As an example, the follow code: 371// 372// type myModule struct { 373// properties struct { 374// Foo string 375// Bar []string 376// } 377// } 378// 379// func NewMyModule() (blueprint.Module, []interface{}) { 380// module := new(myModule) 381// properties := &module.properties 382// return module, []interface{}{properties} 383// } 384// 385// func main() { 386// ctx := blueprint.NewContext() 387// ctx.RegisterModuleType("my_module", NewMyModule) 388// // ... 389// } 390// 391// would support parsing a module defined in a Blueprints file as follows: 392// 393// my_module { 394// name: "myName", 395// foo: "my foo string", 396// bar: ["my", "bar", "strings"], 397// } 398// 399// The factory function may be called from multiple goroutines. Any accesses 400// to global variables must be synchronized. 401func (c *Context) RegisterModuleType(name string, factory ModuleFactory) { 402 if _, present := c.moduleFactories[name]; present { 403 panic(errors.New("module type name is already registered")) 404 } 405 c.moduleFactories[name] = factory 406} 407 408// A SingletonFactory function creates a new Singleton object. See the 409// Context.RegisterSingletonType method for details about how a registered 410// SingletonFactory is used by a Context. 411type SingletonFactory func() Singleton 412 413// RegisterSingletonType registers a singleton type that will be invoked to 414// generate build actions. Each registered singleton type is instantiated and 415// and invoked exactly once as part of the generate phase. Each registered 416// singleton is invoked in registration order. 417// 418// The singleton type names given here must be unique for the context. The 419// factory function should be a named function so that its package and name can 420// be included in the generated Ninja file for debugging purposes. 421func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) { 422 for _, s := range c.singletonInfo { 423 if s.name == name { 424 panic(errors.New("singleton name is already registered")) 425 } 426 } 427 428 c.singletonInfo = append(c.singletonInfo, &singletonInfo{ 429 factory: factory, 430 singleton: factory(), 431 name: name, 432 }) 433} 434 435// RegisterPreSingletonType registers a presingleton type that will be invoked to 436// generate build actions before any Blueprint files have been read. Each registered 437// presingleton type is instantiated and invoked exactly once at the beginning of the 438// parse phase. Each registered presingleton is invoked in registration order. 439// 440// The presingleton type names given here must be unique for the context. The 441// factory function should be a named function so that its package and name can 442// be included in the generated Ninja file for debugging purposes. 443func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory) { 444 for _, s := range c.preSingletonInfo { 445 if s.name == name { 446 panic(errors.New("presingleton name is already registered")) 447 } 448 } 449 450 c.preSingletonInfo = append(c.preSingletonInfo, &singletonInfo{ 451 factory: factory, 452 singleton: factory(), 453 name: name, 454 }) 455} 456 457func (c *Context) SetNameInterface(i NameInterface) { 458 c.nameInterface = i 459} 460 461func (c *Context) SetSrcDir(path string) { 462 c.srcDir = path 463 c.fs = pathtools.NewOsFs(path) 464} 465 466func (c *Context) SrcDir() string { 467 return c.srcDir 468} 469 470func singletonPkgPath(singleton Singleton) string { 471 typ := reflect.TypeOf(singleton) 472 for typ.Kind() == reflect.Ptr { 473 typ = typ.Elem() 474 } 475 return typ.PkgPath() 476} 477 478func singletonTypeName(singleton Singleton) string { 479 typ := reflect.TypeOf(singleton) 480 for typ.Kind() == reflect.Ptr { 481 typ = typ.Elem() 482 } 483 return typ.PkgPath() + "." + typ.Name() 484} 485 486// RegisterTopDownMutator registers a mutator that will be invoked to propagate dependency info 487// top-down between Modules. Each registered mutator is invoked in registration order (mixing 488// TopDownMutators and BottomUpMutators) once per Module, and the invocation on any module will 489// have returned before it is in invoked on any of its dependencies. 490// 491// The mutator type names given here must be unique to all top down mutators in 492// the Context. 493// 494// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 495// parallel while maintaining ordering. 496func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) MutatorHandle { 497 for _, m := range c.mutatorInfo { 498 if m.name == name && m.topDownMutator != nil { 499 panic(fmt.Errorf("mutator name %s is already registered", name)) 500 } 501 } 502 503 info := &mutatorInfo{ 504 topDownMutator: mutator, 505 name: name, 506 } 507 508 c.mutatorInfo = append(c.mutatorInfo, info) 509 510 return info 511} 512 513// RegisterBottomUpMutator registers a mutator that will be invoked to split Modules into variants. 514// Each registered mutator is invoked in registration order (mixing TopDownMutators and 515// BottomUpMutators) once per Module, will not be invoked on a module until the invocations on all 516// of the modules dependencies have returned. 517// 518// The mutator type names given here must be unique to all bottom up or early 519// mutators in the Context. 520// 521// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 522// parallel while maintaining ordering. 523func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) MutatorHandle { 524 for _, m := range c.variantMutatorNames { 525 if m == name { 526 panic(fmt.Errorf("mutator name %s is already registered", name)) 527 } 528 } 529 530 info := &mutatorInfo{ 531 bottomUpMutator: mutator, 532 name: name, 533 } 534 c.mutatorInfo = append(c.mutatorInfo, info) 535 536 c.variantMutatorNames = append(c.variantMutatorNames, name) 537 538 return info 539} 540 541type MutatorHandle interface { 542 // Set the mutator to visit modules in parallel while maintaining ordering. Calling any 543 // method on the mutator context is thread-safe, but the mutator must handle synchronization 544 // for any modifications to global state or any modules outside the one it was invoked on. 545 Parallel() MutatorHandle 546} 547 548func (mutator *mutatorInfo) Parallel() MutatorHandle { 549 mutator.parallel = true 550 return mutator 551} 552 553// RegisterEarlyMutator registers a mutator that will be invoked to split 554// Modules into multiple variant Modules before any dependencies have been 555// created. Each registered mutator is invoked in registration order once 556// per Module (including each variant from previous early mutators). Module 557// order is unpredictable. 558// 559// In order for dependencies to be satisifed in a later pass, all dependencies 560// of a module either must have an identical variant or must have no variations. 561// 562// The mutator type names given here must be unique to all bottom up or early 563// mutators in the Context. 564// 565// Deprecated, use a BottomUpMutator instead. The only difference between 566// EarlyMutator and BottomUpMutator is that EarlyMutator runs before the 567// deprecated DynamicDependencies. 568func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) { 569 for _, m := range c.variantMutatorNames { 570 if m == name { 571 panic(fmt.Errorf("mutator name %s is already registered", name)) 572 } 573 } 574 575 c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{ 576 bottomUpMutator: func(mctx BottomUpMutatorContext) { 577 mutator(mctx) 578 }, 579 name: name, 580 }) 581 582 c.variantMutatorNames = append(c.variantMutatorNames, name) 583} 584 585// SetIgnoreUnknownModuleTypes sets the behavior of the context in the case 586// where it encounters an unknown module type while parsing Blueprints files. By 587// default, the context will report unknown module types as an error. If this 588// method is called with ignoreUnknownModuleTypes set to true then the context 589// will silently ignore unknown module types. 590// 591// This method should generally not be used. It exists to facilitate the 592// bootstrapping process. 593func (c *Context) SetIgnoreUnknownModuleTypes(ignoreUnknownModuleTypes bool) { 594 c.ignoreUnknownModuleTypes = ignoreUnknownModuleTypes 595} 596 597// SetAllowMissingDependencies changes the behavior of Blueprint to ignore 598// unresolved dependencies. If the module's GenerateBuildActions calls 599// ModuleContext.GetMissingDependencies Blueprint will not emit any errors 600// for missing dependencies. 601func (c *Context) SetAllowMissingDependencies(allowMissingDependencies bool) { 602 c.allowMissingDependencies = allowMissingDependencies 603} 604 605func (c *Context) SetModuleListFile(listFile string) { 606 c.moduleListFile = listFile 607} 608 609func (c *Context) ListModulePaths(baseDir string) (paths []string, err error) { 610 reader, err := c.fs.Open(c.moduleListFile) 611 if err != nil { 612 return nil, err 613 } 614 bytes, err := ioutil.ReadAll(reader) 615 if err != nil { 616 return nil, err 617 } 618 text := string(bytes) 619 620 text = strings.Trim(text, "\n") 621 lines := strings.Split(text, "\n") 622 for i := range lines { 623 lines[i] = filepath.Join(baseDir, lines[i]) 624 } 625 626 return lines, nil 627} 628 629// a fileParseContext tells the status of parsing a particular file 630type fileParseContext struct { 631 // name of file 632 fileName string 633 634 // scope to use when resolving variables 635 Scope *parser.Scope 636 637 // pointer to the one in the parent directory 638 parent *fileParseContext 639 640 // is closed once FileHandler has completed for this file 641 doneVisiting chan struct{} 642} 643 644// ParseBlueprintsFiles parses a set of Blueprints files starting with the file 645// at rootFile. When it encounters a Blueprints file with a set of subdirs 646// listed it recursively parses any Blueprints files found in those 647// subdirectories. 648// 649// If no errors are encountered while parsing the files, the list of paths on 650// which the future output will depend is returned. This list will include both 651// Blueprints file paths as well as directory paths for cases where wildcard 652// subdirs are found. 653func (c *Context) ParseBlueprintsFiles(rootFile string, 654 config interface{}) (deps []string, errs []error) { 655 656 baseDir := filepath.Dir(rootFile) 657 pathsToParse, err := c.ListModulePaths(baseDir) 658 if err != nil { 659 return nil, []error{err} 660 } 661 return c.ParseFileList(baseDir, pathsToParse, config) 662} 663 664func (c *Context) ParseFileList(rootDir string, filePaths []string, 665 config interface{}) (deps []string, errs []error) { 666 667 if len(filePaths) < 1 { 668 return nil, []error{fmt.Errorf("no paths provided to parse")} 669 } 670 671 c.dependenciesReady = false 672 673 type newModuleInfo struct { 674 *moduleInfo 675 added chan<- struct{} 676 } 677 678 moduleCh := make(chan newModuleInfo) 679 errsCh := make(chan []error) 680 doneCh := make(chan struct{}) 681 var numErrs uint32 682 var numGoroutines int32 683 684 // handler must be reentrant 685 handleOneFile := func(file *parser.File) { 686 if atomic.LoadUint32(&numErrs) > maxErrors { 687 return 688 } 689 690 addedCh := make(chan struct{}) 691 692 var scopedModuleFactories map[string]ModuleFactory 693 694 var addModule func(module *moduleInfo) []error 695 addModule = func(module *moduleInfo) []error { 696 // Run any load hooks immediately before it is sent to the moduleCh and is 697 // registered by name. This allows load hooks to set and/or modify any aspect 698 // of the module (including names) using information that is not available when 699 // the module factory is called. 700 newModules, errs := runAndRemoveLoadHooks(c, config, module, &scopedModuleFactories) 701 if len(errs) > 0 { 702 return errs 703 } 704 705 moduleCh <- newModuleInfo{module, addedCh} 706 <-addedCh 707 for _, n := range newModules { 708 errs = addModule(n) 709 if len(errs) > 0 { 710 return errs 711 } 712 } 713 return nil 714 } 715 716 for _, def := range file.Defs { 717 switch def := def.(type) { 718 case *parser.Module: 719 module, errs := processModuleDef(def, file.Name, c.moduleFactories, scopedModuleFactories, c.ignoreUnknownModuleTypes) 720 if len(errs) == 0 && module != nil { 721 errs = addModule(module) 722 } 723 724 if len(errs) > 0 { 725 atomic.AddUint32(&numErrs, uint32(len(errs))) 726 errsCh <- errs 727 } 728 729 case *parser.Assignment: 730 // Already handled via Scope object 731 default: 732 panic("unknown definition type") 733 } 734 735 } 736 } 737 738 atomic.AddInt32(&numGoroutines, 1) 739 go func() { 740 var errs []error 741 deps, errs = c.WalkBlueprintsFiles(rootDir, filePaths, handleOneFile) 742 if len(errs) > 0 { 743 errsCh <- errs 744 } 745 doneCh <- struct{}{} 746 }() 747 748loop: 749 for { 750 select { 751 case newErrs := <-errsCh: 752 errs = append(errs, newErrs...) 753 case module := <-moduleCh: 754 newErrs := c.addModule(module.moduleInfo) 755 if module.added != nil { 756 module.added <- struct{}{} 757 } 758 if len(newErrs) > 0 { 759 errs = append(errs, newErrs...) 760 } 761 case <-doneCh: 762 n := atomic.AddInt32(&numGoroutines, -1) 763 if n == 0 { 764 break loop 765 } 766 } 767 } 768 769 return deps, errs 770} 771 772type FileHandler func(*parser.File) 773 774// WalkBlueprintsFiles walks a set of Blueprints files starting with the given filepaths, 775// calling the given file handler on each 776// 777// When WalkBlueprintsFiles encounters a Blueprints file with a set of subdirs listed, 778// it recursively parses any Blueprints files found in those subdirectories. 779// 780// If any of the file paths is an ancestor directory of any other of file path, the ancestor 781// will be parsed and visited first. 782// 783// the file handler will be called from a goroutine, so it must be reentrant. 784// 785// If no errors are encountered while parsing the files, the list of paths on 786// which the future output will depend is returned. This list will include both 787// Blueprints file paths as well as directory paths for cases where wildcard 788// subdirs are found. 789// 790// visitor will be called asynchronously, and will only be called once visitor for each 791// ancestor directory has completed. 792// 793// WalkBlueprintsFiles will not return until all calls to visitor have returned. 794func (c *Context) WalkBlueprintsFiles(rootDir string, filePaths []string, 795 visitor FileHandler) (deps []string, errs []error) { 796 797 // make a mapping from ancestors to their descendants to facilitate parsing ancestors first 798 descendantsMap, err := findBlueprintDescendants(filePaths) 799 if err != nil { 800 panic(err.Error()) 801 } 802 blueprintsSet := make(map[string]bool) 803 804 // Channels to receive data back from openAndParse goroutines 805 blueprintsCh := make(chan fileParseContext) 806 errsCh := make(chan []error) 807 depsCh := make(chan string) 808 809 // Channel to notify main loop that a openAndParse goroutine has finished 810 doneParsingCh := make(chan fileParseContext) 811 812 // Number of outstanding goroutines to wait for 813 activeCount := 0 814 var pending []fileParseContext 815 tooManyErrors := false 816 817 // Limit concurrent calls to parseBlueprintFiles to 200 818 // Darwin has a default limit of 256 open files 819 maxActiveCount := 200 820 821 // count the number of pending calls to visitor() 822 visitorWaitGroup := sync.WaitGroup{} 823 824 startParseBlueprintsFile := func(blueprint fileParseContext) { 825 if blueprintsSet[blueprint.fileName] { 826 return 827 } 828 blueprintsSet[blueprint.fileName] = true 829 activeCount++ 830 deps = append(deps, blueprint.fileName) 831 visitorWaitGroup.Add(1) 832 go func() { 833 file, blueprints, deps, errs := c.openAndParse(blueprint.fileName, blueprint.Scope, rootDir, 834 &blueprint) 835 if len(errs) > 0 { 836 errsCh <- errs 837 } 838 for _, blueprint := range blueprints { 839 blueprintsCh <- blueprint 840 } 841 for _, dep := range deps { 842 depsCh <- dep 843 } 844 doneParsingCh <- blueprint 845 846 if blueprint.parent != nil && blueprint.parent.doneVisiting != nil { 847 // wait for visitor() of parent to complete 848 <-blueprint.parent.doneVisiting 849 } 850 851 if len(errs) == 0 { 852 // process this file 853 visitor(file) 854 } 855 if blueprint.doneVisiting != nil { 856 close(blueprint.doneVisiting) 857 } 858 visitorWaitGroup.Done() 859 }() 860 } 861 862 foundParseableBlueprint := func(blueprint fileParseContext) { 863 if activeCount >= maxActiveCount { 864 pending = append(pending, blueprint) 865 } else { 866 startParseBlueprintsFile(blueprint) 867 } 868 } 869 870 startParseDescendants := func(blueprint fileParseContext) { 871 descendants, hasDescendants := descendantsMap[blueprint.fileName] 872 if hasDescendants { 873 for _, descendant := range descendants { 874 foundParseableBlueprint(fileParseContext{descendant, parser.NewScope(blueprint.Scope), &blueprint, make(chan struct{})}) 875 } 876 } 877 } 878 879 // begin parsing any files that have no ancestors 880 startParseDescendants(fileParseContext{"", parser.NewScope(nil), nil, nil}) 881 882loop: 883 for { 884 if len(errs) > maxErrors { 885 tooManyErrors = true 886 } 887 888 select { 889 case newErrs := <-errsCh: 890 errs = append(errs, newErrs...) 891 case dep := <-depsCh: 892 deps = append(deps, dep) 893 case blueprint := <-blueprintsCh: 894 if tooManyErrors { 895 continue 896 } 897 foundParseableBlueprint(blueprint) 898 case blueprint := <-doneParsingCh: 899 activeCount-- 900 if !tooManyErrors { 901 startParseDescendants(blueprint) 902 } 903 if activeCount < maxActiveCount && len(pending) > 0 { 904 // start to process the next one from the queue 905 next := pending[len(pending)-1] 906 pending = pending[:len(pending)-1] 907 startParseBlueprintsFile(next) 908 } 909 if activeCount == 0 { 910 break loop 911 } 912 } 913 } 914 915 sort.Strings(deps) 916 917 // wait for every visitor() to complete 918 visitorWaitGroup.Wait() 919 920 return 921} 922 923// MockFileSystem causes the Context to replace all reads with accesses to the provided map of 924// filenames to contents stored as a byte slice. 925func (c *Context) MockFileSystem(files map[string][]byte) { 926 // look for a module list file 927 _, ok := files[MockModuleListFile] 928 if !ok { 929 // no module list file specified; find every file named Blueprints 930 pathsToParse := []string{} 931 for candidate := range files { 932 if filepath.Base(candidate) == "Blueprints" { 933 pathsToParse = append(pathsToParse, candidate) 934 } 935 } 936 if len(pathsToParse) < 1 { 937 panic(fmt.Sprintf("No Blueprints files found in mock filesystem: %v\n", files)) 938 } 939 // put the list of Blueprints files into a list file 940 files[MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n")) 941 } 942 c.SetModuleListFile(MockModuleListFile) 943 944 // mock the filesystem 945 c.fs = pathtools.MockFs(files) 946} 947 948func (c *Context) SetFs(fs pathtools.FileSystem) { 949 c.fs = fs 950} 951 952// openAndParse opens and parses a single Blueprints file, and returns the results 953func (c *Context) openAndParse(filename string, scope *parser.Scope, rootDir string, 954 parent *fileParseContext) (file *parser.File, 955 subBlueprints []fileParseContext, deps []string, errs []error) { 956 957 f, err := c.fs.Open(filename) 958 if err != nil { 959 // couldn't open the file; see if we can provide a clearer error than "could not open file" 960 stats, statErr := c.fs.Lstat(filename) 961 if statErr == nil { 962 isSymlink := stats.Mode()&os.ModeSymlink != 0 963 if isSymlink { 964 err = fmt.Errorf("could not open symlink %v : %v", filename, err) 965 target, readlinkErr := os.Readlink(filename) 966 if readlinkErr == nil { 967 _, targetStatsErr := c.fs.Lstat(target) 968 if targetStatsErr != nil { 969 err = fmt.Errorf("could not open symlink %v; its target (%v) cannot be opened", filename, target) 970 } 971 } 972 } else { 973 err = fmt.Errorf("%v exists but could not be opened: %v", filename, err) 974 } 975 } 976 return nil, nil, nil, []error{err} 977 } 978 979 func() { 980 defer func() { 981 err = f.Close() 982 if err != nil { 983 errs = append(errs, err) 984 } 985 }() 986 file, subBlueprints, errs = c.parseOne(rootDir, filename, f, scope, parent) 987 }() 988 989 if len(errs) > 0 { 990 return nil, nil, nil, errs 991 } 992 993 for _, b := range subBlueprints { 994 deps = append(deps, b.fileName) 995 } 996 997 return file, subBlueprints, deps, nil 998} 999 1000// parseOne parses a single Blueprints file from the given reader, creating Module 1001// objects for each of the module definitions encountered. If the Blueprints 1002// file contains an assignment to the "subdirs" variable, then the 1003// subdirectories listed are searched for Blueprints files returned in the 1004// subBlueprints return value. If the Blueprints file contains an assignment 1005// to the "build" variable, then the file listed are returned in the 1006// subBlueprints return value. 1007// 1008// rootDir specifies the path to the root directory of the source tree, while 1009// filename specifies the path to the Blueprints file. These paths are used for 1010// error reporting and for determining the module's directory. 1011func (c *Context) parseOne(rootDir, filename string, reader io.Reader, 1012 scope *parser.Scope, parent *fileParseContext) (file *parser.File, subBlueprints []fileParseContext, errs []error) { 1013 1014 relBlueprintsFile, err := filepath.Rel(rootDir, filename) 1015 if err != nil { 1016 return nil, nil, []error{err} 1017 } 1018 1019 scope.Remove("subdirs") 1020 scope.Remove("optional_subdirs") 1021 scope.Remove("build") 1022 file, errs = parser.ParseAndEval(filename, reader, scope) 1023 if len(errs) > 0 { 1024 for i, err := range errs { 1025 if parseErr, ok := err.(*parser.ParseError); ok { 1026 err = &BlueprintError{ 1027 Err: parseErr.Err, 1028 Pos: parseErr.Pos, 1029 } 1030 errs[i] = err 1031 } 1032 } 1033 1034 // If there were any parse errors don't bother trying to interpret the 1035 // result. 1036 return nil, nil, errs 1037 } 1038 file.Name = relBlueprintsFile 1039 1040 build, buildPos, err := getLocalStringListFromScope(scope, "build") 1041 if err != nil { 1042 errs = append(errs, err) 1043 } 1044 for _, buildEntry := range build { 1045 if strings.Contains(buildEntry, "/") { 1046 errs = append(errs, &BlueprintError{ 1047 Err: fmt.Errorf("illegal value %v. The '/' character is not permitted", buildEntry), 1048 Pos: buildPos, 1049 }) 1050 } 1051 } 1052 1053 subBlueprintsName, _, err := getStringFromScope(scope, "subname") 1054 if err != nil { 1055 errs = append(errs, err) 1056 } 1057 1058 if subBlueprintsName == "" { 1059 subBlueprintsName = "Blueprints" 1060 } 1061 1062 var blueprints []string 1063 1064 newBlueprints, newErrs := c.findBuildBlueprints(filepath.Dir(filename), build, buildPos) 1065 blueprints = append(blueprints, newBlueprints...) 1066 errs = append(errs, newErrs...) 1067 1068 subBlueprintsAndScope := make([]fileParseContext, len(blueprints)) 1069 for i, b := range blueprints { 1070 subBlueprintsAndScope[i] = fileParseContext{b, parser.NewScope(scope), parent, make(chan struct{})} 1071 } 1072 return file, subBlueprintsAndScope, errs 1073} 1074 1075func (c *Context) findBuildBlueprints(dir string, build []string, 1076 buildPos scanner.Position) ([]string, []error) { 1077 1078 var blueprints []string 1079 var errs []error 1080 1081 for _, file := range build { 1082 pattern := filepath.Join(dir, file) 1083 var matches []string 1084 var err error 1085 1086 matches, err = c.glob(pattern, nil) 1087 1088 if err != nil { 1089 errs = append(errs, &BlueprintError{ 1090 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1091 Pos: buildPos, 1092 }) 1093 continue 1094 } 1095 1096 if len(matches) == 0 { 1097 errs = append(errs, &BlueprintError{ 1098 Err: fmt.Errorf("%q: not found", pattern), 1099 Pos: buildPos, 1100 }) 1101 } 1102 1103 for _, foundBlueprints := range matches { 1104 if strings.HasSuffix(foundBlueprints, "/") { 1105 errs = append(errs, &BlueprintError{ 1106 Err: fmt.Errorf("%q: is a directory", foundBlueprints), 1107 Pos: buildPos, 1108 }) 1109 } 1110 blueprints = append(blueprints, foundBlueprints) 1111 } 1112 } 1113 1114 return blueprints, errs 1115} 1116 1117func (c *Context) findSubdirBlueprints(dir string, subdirs []string, subdirsPos scanner.Position, 1118 subBlueprintsName string, optional bool) ([]string, []error) { 1119 1120 var blueprints []string 1121 var errs []error 1122 1123 for _, subdir := range subdirs { 1124 pattern := filepath.Join(dir, subdir, subBlueprintsName) 1125 var matches []string 1126 var err error 1127 1128 matches, err = c.glob(pattern, nil) 1129 1130 if err != nil { 1131 errs = append(errs, &BlueprintError{ 1132 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1133 Pos: subdirsPos, 1134 }) 1135 continue 1136 } 1137 1138 if len(matches) == 0 && !optional { 1139 errs = append(errs, &BlueprintError{ 1140 Err: fmt.Errorf("%q: not found", pattern), 1141 Pos: subdirsPos, 1142 }) 1143 } 1144 1145 for _, subBlueprints := range matches { 1146 if strings.HasSuffix(subBlueprints, "/") { 1147 errs = append(errs, &BlueprintError{ 1148 Err: fmt.Errorf("%q: is a directory", subBlueprints), 1149 Pos: subdirsPos, 1150 }) 1151 } 1152 blueprints = append(blueprints, subBlueprints) 1153 } 1154 } 1155 1156 return blueprints, errs 1157} 1158 1159func getLocalStringListFromScope(scope *parser.Scope, v string) ([]string, scanner.Position, error) { 1160 if assignment, local := scope.Get(v); assignment == nil || !local { 1161 return nil, scanner.Position{}, nil 1162 } else { 1163 switch value := assignment.Value.Eval().(type) { 1164 case *parser.List: 1165 ret := make([]string, 0, len(value.Values)) 1166 1167 for _, listValue := range value.Values { 1168 s, ok := listValue.(*parser.String) 1169 if !ok { 1170 // The parser should not produce this. 1171 panic("non-string value found in list") 1172 } 1173 1174 ret = append(ret, s.Value) 1175 } 1176 1177 return ret, assignment.EqualsPos, nil 1178 case *parser.Bool, *parser.String: 1179 return nil, scanner.Position{}, &BlueprintError{ 1180 Err: fmt.Errorf("%q must be a list of strings", v), 1181 Pos: assignment.EqualsPos, 1182 } 1183 default: 1184 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1185 } 1186 } 1187} 1188 1189func getStringFromScope(scope *parser.Scope, v string) (string, scanner.Position, error) { 1190 if assignment, _ := scope.Get(v); assignment == nil { 1191 return "", scanner.Position{}, nil 1192 } else { 1193 switch value := assignment.Value.Eval().(type) { 1194 case *parser.String: 1195 return value.Value, assignment.EqualsPos, nil 1196 case *parser.Bool, *parser.List: 1197 return "", scanner.Position{}, &BlueprintError{ 1198 Err: fmt.Errorf("%q must be a string", v), 1199 Pos: assignment.EqualsPos, 1200 } 1201 default: 1202 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1203 } 1204 } 1205} 1206 1207// Clones a build logic module by calling the factory method for its module type, and then cloning 1208// property values. Any values stored in the module object that are not stored in properties 1209// structs will be lost. 1210func (c *Context) cloneLogicModule(origModule *moduleInfo) (Module, []interface{}) { 1211 newLogicModule, newProperties := origModule.factory() 1212 1213 if len(newProperties) != len(origModule.properties) { 1214 panic("mismatched properties array length in " + origModule.Name()) 1215 } 1216 1217 for i := range newProperties { 1218 dst := reflect.ValueOf(newProperties[i]) 1219 src := reflect.ValueOf(origModule.properties[i]) 1220 1221 proptools.CopyProperties(dst, src) 1222 } 1223 1224 return newLogicModule, newProperties 1225} 1226 1227func (c *Context) createVariations(origModule *moduleInfo, mutatorName string, 1228 defaultVariationName *string, variationNames []string) ([]*moduleInfo, []error) { 1229 1230 if len(variationNames) == 0 { 1231 panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q", 1232 mutatorName, origModule.Name())) 1233 } 1234 1235 newModules := []*moduleInfo{} 1236 1237 var errs []error 1238 1239 for i, variationName := range variationNames { 1240 var newLogicModule Module 1241 var newProperties []interface{} 1242 1243 if i == 0 { 1244 // Reuse the existing module for the first new variant 1245 // This both saves creating a new module, and causes the insertion in c.moduleInfo below 1246 // with logicModule as the key to replace the original entry in c.moduleInfo 1247 newLogicModule, newProperties = origModule.logicModule, origModule.properties 1248 } else { 1249 newLogicModule, newProperties = c.cloneLogicModule(origModule) 1250 } 1251 1252 newVariant := origModule.variant.clone() 1253 if newVariant == nil { 1254 newVariant = make(variationMap) 1255 } 1256 newVariant[mutatorName] = variationName 1257 1258 m := *origModule 1259 newModule := &m 1260 newModule.directDeps = append([]depInfo{}, origModule.directDeps...) 1261 newModule.logicModule = newLogicModule 1262 newModule.variant = newVariant 1263 newModule.dependencyVariant = origModule.dependencyVariant.clone() 1264 newModule.properties = newProperties 1265 1266 if variationName != "" { 1267 if newModule.variantName == "" { 1268 newModule.variantName = variationName 1269 } else { 1270 newModule.variantName += "_" + variationName 1271 } 1272 } 1273 1274 newModules = append(newModules, newModule) 1275 1276 newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName, defaultVariationName) 1277 if len(newErrs) > 0 { 1278 errs = append(errs, newErrs...) 1279 } 1280 } 1281 1282 // Mark original variant as invalid. Modules that depend on this module will still 1283 // depend on origModule, but we'll fix it when the mutator is called on them. 1284 origModule.logicModule = nil 1285 origModule.splitModules = newModules 1286 1287 atomic.AddUint32(&c.depsModified, 1) 1288 1289 return newModules, errs 1290} 1291 1292func (c *Context) convertDepsToVariation(module *moduleInfo, 1293 mutatorName, variationName string, defaultVariationName *string) (errs []error) { 1294 1295 for i, dep := range module.directDeps { 1296 if dep.module.logicModule == nil { 1297 var newDep *moduleInfo 1298 for _, m := range dep.module.splitModules { 1299 if m.variant[mutatorName] == variationName { 1300 newDep = m 1301 break 1302 } 1303 } 1304 if newDep == nil && defaultVariationName != nil { 1305 // give it a second chance; match with defaultVariationName 1306 for _, m := range dep.module.splitModules { 1307 if m.variant[mutatorName] == *defaultVariationName { 1308 newDep = m 1309 break 1310 } 1311 } 1312 } 1313 if newDep == nil { 1314 errs = append(errs, &BlueprintError{ 1315 Err: fmt.Errorf("failed to find variation %q for module %q needed by %q", 1316 variationName, dep.module.Name(), module.Name()), 1317 Pos: module.pos, 1318 }) 1319 continue 1320 } 1321 module.directDeps[i].module = newDep 1322 } 1323 } 1324 1325 return errs 1326} 1327 1328func (c *Context) prettyPrintVariant(variant variationMap) string { 1329 names := make([]string, 0, len(variant)) 1330 for _, m := range c.variantMutatorNames { 1331 if v, ok := variant[m]; ok { 1332 names = append(names, m+":"+v) 1333 } 1334 } 1335 1336 return strings.Join(names, ", ") 1337} 1338 1339func (c *Context) prettyPrintGroupVariants(group *moduleGroup) string { 1340 var variants []string 1341 for _, mod := range group.modules { 1342 variants = append(variants, c.prettyPrintVariant(mod.variant)) 1343 } 1344 for _, mod := range group.aliases { 1345 variants = append(variants, c.prettyPrintVariant(mod.variant)+ 1346 "(alias to "+c.prettyPrintVariant(mod.target.variant)+")") 1347 } 1348 sort.Strings(variants) 1349 return strings.Join(variants, "\n ") 1350} 1351 1352func newModule(factory ModuleFactory) *moduleInfo { 1353 logicModule, properties := factory() 1354 1355 module := &moduleInfo{ 1356 logicModule: logicModule, 1357 factory: factory, 1358 } 1359 1360 module.properties = properties 1361 1362 return module 1363} 1364 1365func processModuleDef(moduleDef *parser.Module, 1366 relBlueprintsFile string, moduleFactories, scopedModuleFactories map[string]ModuleFactory, ignoreUnknownModuleTypes bool) (*moduleInfo, []error) { 1367 1368 factory, ok := moduleFactories[moduleDef.Type] 1369 if !ok && scopedModuleFactories != nil { 1370 factory, ok = scopedModuleFactories[moduleDef.Type] 1371 } 1372 if !ok { 1373 if ignoreUnknownModuleTypes { 1374 return nil, nil 1375 } 1376 1377 return nil, []error{ 1378 &BlueprintError{ 1379 Err: fmt.Errorf("unrecognized module type %q", moduleDef.Type), 1380 Pos: moduleDef.TypePos, 1381 }, 1382 } 1383 } 1384 1385 module := newModule(factory) 1386 module.typeName = moduleDef.Type 1387 1388 module.relBlueprintsFile = relBlueprintsFile 1389 1390 propertyMap, errs := proptools.UnpackProperties(moduleDef.Properties, module.properties...) 1391 if len(errs) > 0 { 1392 for i, err := range errs { 1393 if unpackErr, ok := err.(*proptools.UnpackError); ok { 1394 err = &BlueprintError{ 1395 Err: unpackErr.Err, 1396 Pos: unpackErr.Pos, 1397 } 1398 errs[i] = err 1399 } 1400 } 1401 return nil, errs 1402 } 1403 1404 module.pos = moduleDef.TypePos 1405 module.propertyPos = make(map[string]scanner.Position) 1406 for name, propertyDef := range propertyMap { 1407 module.propertyPos[name] = propertyDef.ColonPos 1408 } 1409 1410 return module, nil 1411} 1412 1413func (c *Context) addModule(module *moduleInfo) []error { 1414 name := module.logicModule.Name() 1415 if name == "" { 1416 return []error{ 1417 &BlueprintError{ 1418 Err: fmt.Errorf("property 'name' is missing from a module"), 1419 Pos: module.pos, 1420 }, 1421 } 1422 } 1423 c.moduleInfo[module.logicModule] = module 1424 1425 group := &moduleGroup{ 1426 name: name, 1427 modules: []*moduleInfo{module}, 1428 } 1429 module.group = group 1430 namespace, errs := c.nameInterface.NewModule( 1431 newNamespaceContext(module), 1432 ModuleGroup{moduleGroup: group}, 1433 module.logicModule) 1434 if len(errs) > 0 { 1435 for i := range errs { 1436 errs[i] = &BlueprintError{Err: errs[i], Pos: module.pos} 1437 } 1438 return errs 1439 } 1440 group.namespace = namespace 1441 1442 c.moduleGroups = append(c.moduleGroups, group) 1443 1444 return nil 1445} 1446 1447// ResolveDependencies checks that the dependencies specified by all of the 1448// modules defined in the parsed Blueprints files are valid. This means that 1449// the modules depended upon are defined and that no circular dependencies 1450// exist. 1451func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) { 1452 return c.resolveDependencies(c.Context, config) 1453} 1454 1455func (c *Context) resolveDependencies(ctx context.Context, config interface{}) (deps []string, errs []error) { 1456 pprof.Do(ctx, pprof.Labels("blueprint", "ResolveDependencies"), func(ctx context.Context) { 1457 c.liveGlobals = newLiveTracker(config) 1458 1459 deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals) 1460 if len(errs) > 0 { 1461 return 1462 } 1463 1464 errs = c.updateDependencies() 1465 if len(errs) > 0 { 1466 return 1467 } 1468 1469 var mutatorDeps []string 1470 mutatorDeps, errs = c.runMutators(ctx, config) 1471 if len(errs) > 0 { 1472 return 1473 } 1474 deps = append(deps, mutatorDeps...) 1475 1476 c.cloneModules() 1477 1478 c.dependenciesReady = true 1479 }) 1480 1481 if len(errs) > 0 { 1482 return nil, errs 1483 } 1484 1485 return deps, nil 1486} 1487 1488// Default dependencies handling. If the module implements the (deprecated) 1489// DynamicDependerModule interface then this set consists of the union of those 1490// module names returned by its DynamicDependencies method and those added by calling 1491// AddDependencies or AddVariationDependencies on DynamicDependencyModuleContext. 1492func blueprintDepsMutator(ctx BottomUpMutatorContext) { 1493 if dynamicDepender, ok := ctx.Module().(DynamicDependerModule); ok { 1494 func() { 1495 defer func() { 1496 if r := recover(); r != nil { 1497 ctx.error(newPanicErrorf(r, "DynamicDependencies for %s", ctx.moduleInfo())) 1498 } 1499 }() 1500 dynamicDeps := dynamicDepender.DynamicDependencies(ctx) 1501 1502 if ctx.Failed() { 1503 return 1504 } 1505 1506 ctx.AddDependency(ctx.Module(), nil, dynamicDeps...) 1507 }() 1508 } 1509} 1510 1511// findMatchingVariant searches the moduleGroup for a module with the same variant as module, 1512// and returns the matching module, or nil if one is not found. 1513func (c *Context) findMatchingVariant(module *moduleInfo, possible *moduleGroup, reverse bool) *moduleInfo { 1514 if len(possible.modules) == 1 { 1515 return possible.modules[0] 1516 } else { 1517 var variantToMatch variationMap 1518 if !reverse { 1519 // For forward dependency, ignore local variants by matching against 1520 // dependencyVariant which doesn't have the local variants 1521 variantToMatch = module.dependencyVariant 1522 } else { 1523 // For reverse dependency, use all the variants 1524 variantToMatch = module.variant 1525 } 1526 for _, m := range possible.modules { 1527 if m.variant.equal(variantToMatch) { 1528 return m 1529 } 1530 } 1531 for _, m := range possible.aliases { 1532 if m.variant.equal(variantToMatch) { 1533 return m.target 1534 } 1535 } 1536 } 1537 1538 return nil 1539} 1540 1541func (c *Context) addDependency(module *moduleInfo, tag DependencyTag, depName string) []error { 1542 if _, ok := tag.(BaseDependencyTag); ok { 1543 panic("BaseDependencyTag is not allowed to be used directly!") 1544 } 1545 1546 if depName == module.Name() { 1547 return []error{&BlueprintError{ 1548 Err: fmt.Errorf("%q depends on itself", depName), 1549 Pos: module.pos, 1550 }} 1551 } 1552 1553 possibleDeps := c.moduleGroupFromName(depName, module.namespace()) 1554 if possibleDeps == nil { 1555 return c.discoveredMissingDependencies(module, depName) 1556 } 1557 1558 if m := c.findMatchingVariant(module, possibleDeps, false); m != nil { 1559 module.newDirectDeps = append(module.newDirectDeps, depInfo{m, tag}) 1560 atomic.AddUint32(&c.depsModified, 1) 1561 return nil 1562 } 1563 1564 if c.allowMissingDependencies { 1565 // Allow missing variants. 1566 return c.discoveredMissingDependencies(module, depName+c.prettyPrintVariant(module.dependencyVariant)) 1567 } 1568 1569 return []error{&BlueprintError{ 1570 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1571 depName, module.Name(), 1572 c.prettyPrintVariant(module.dependencyVariant), 1573 c.prettyPrintGroupVariants(possibleDeps)), 1574 Pos: module.pos, 1575 }} 1576} 1577 1578func (c *Context) findReverseDependency(module *moduleInfo, destName string) (*moduleInfo, []error) { 1579 if destName == module.Name() { 1580 return nil, []error{&BlueprintError{ 1581 Err: fmt.Errorf("%q depends on itself", destName), 1582 Pos: module.pos, 1583 }} 1584 } 1585 1586 possibleDeps := c.moduleGroupFromName(destName, module.namespace()) 1587 if possibleDeps == nil { 1588 return nil, []error{&BlueprintError{ 1589 Err: fmt.Errorf("%q has a reverse dependency on undefined module %q", 1590 module.Name(), destName), 1591 Pos: module.pos, 1592 }} 1593 } 1594 1595 if m := c.findMatchingVariant(module, possibleDeps, true); m != nil { 1596 return m, nil 1597 } 1598 1599 if c.allowMissingDependencies { 1600 // Allow missing variants. 1601 return module, c.discoveredMissingDependencies(module, destName+c.prettyPrintVariant(module.dependencyVariant)) 1602 } 1603 1604 return nil, []error{&BlueprintError{ 1605 Err: fmt.Errorf("reverse dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1606 destName, module.Name(), 1607 c.prettyPrintVariant(module.dependencyVariant), 1608 c.prettyPrintGroupVariants(possibleDeps)), 1609 Pos: module.pos, 1610 }} 1611} 1612 1613func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation, 1614 tag DependencyTag, depName string, far bool) []error { 1615 if _, ok := tag.(BaseDependencyTag); ok { 1616 panic("BaseDependencyTag is not allowed to be used directly!") 1617 } 1618 1619 possibleDeps := c.moduleGroupFromName(depName, module.namespace()) 1620 if possibleDeps == nil { 1621 return c.discoveredMissingDependencies(module, depName) 1622 } 1623 1624 // We can't just append variant.Variant to module.dependencyVariants.variantName and 1625 // compare the strings because the result won't be in mutator registration order. 1626 // Create a new map instead, and then deep compare the maps. 1627 var newVariant variationMap 1628 if !far { 1629 newVariant = module.dependencyVariant.clone() 1630 } 1631 for _, v := range variations { 1632 if newVariant == nil { 1633 newVariant = make(variationMap) 1634 } 1635 newVariant[v.Mutator] = v.Variation 1636 } 1637 1638 check := func(variant variationMap) bool { 1639 if far { 1640 return variant.subset(newVariant) 1641 } else { 1642 return variant.equal(newVariant) 1643 } 1644 } 1645 1646 var foundDep *moduleInfo 1647 for _, m := range possibleDeps.modules { 1648 if check(m.variant) { 1649 foundDep = m 1650 break 1651 } 1652 } 1653 1654 if foundDep == nil { 1655 for _, m := range possibleDeps.aliases { 1656 if check(m.variant) { 1657 foundDep = m.target 1658 break 1659 } 1660 } 1661 } 1662 1663 if foundDep == nil { 1664 if c.allowMissingDependencies { 1665 // Allow missing variants. 1666 return c.discoveredMissingDependencies(module, depName+c.prettyPrintVariant(newVariant)) 1667 } 1668 return []error{&BlueprintError{ 1669 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1670 depName, module.Name(), 1671 c.prettyPrintVariant(newVariant), 1672 c.prettyPrintGroupVariants(possibleDeps)), 1673 Pos: module.pos, 1674 }} 1675 } 1676 1677 if module == foundDep { 1678 return []error{&BlueprintError{ 1679 Err: fmt.Errorf("%q depends on itself", depName), 1680 Pos: module.pos, 1681 }} 1682 } 1683 // AddVariationDependency allows adding a dependency on itself, but only if 1684 // that module is earlier in the module list than this one, since we always 1685 // run GenerateBuildActions in order for the variants of a module 1686 if foundDep.group == module.group && beforeInModuleList(module, foundDep, module.group.modules) { 1687 return []error{&BlueprintError{ 1688 Err: fmt.Errorf("%q depends on later version of itself", depName), 1689 Pos: module.pos, 1690 }} 1691 } 1692 module.newDirectDeps = append(module.newDirectDeps, depInfo{foundDep, tag}) 1693 atomic.AddUint32(&c.depsModified, 1) 1694 return nil 1695} 1696 1697func (c *Context) addInterVariantDependency(origModule *moduleInfo, tag DependencyTag, 1698 from, to Module) { 1699 if _, ok := tag.(BaseDependencyTag); ok { 1700 panic("BaseDependencyTag is not allowed to be used directly!") 1701 } 1702 1703 var fromInfo, toInfo *moduleInfo 1704 for _, m := range origModule.splitModules { 1705 if m.logicModule == from { 1706 fromInfo = m 1707 } 1708 if m.logicModule == to { 1709 toInfo = m 1710 if fromInfo != nil { 1711 panic(fmt.Errorf("%q depends on later version of itself", origModule.Name())) 1712 } 1713 } 1714 } 1715 1716 if fromInfo == nil || toInfo == nil { 1717 panic(fmt.Errorf("AddInterVariantDependency called for module %q on invalid variant", 1718 origModule.Name())) 1719 } 1720 1721 fromInfo.newDirectDeps = append(fromInfo.newDirectDeps, depInfo{toInfo, tag}) 1722 atomic.AddUint32(&c.depsModified, 1) 1723} 1724 1725// findBlueprintDescendants returns a map linking parent Blueprints files to child Blueprints files 1726// For example, if paths = []string{"a/b/c/Android.bp", "a/Blueprints"}, 1727// then descendants = {"":[]string{"a/Blueprints"}, "a/Blueprints":[]string{"a/b/c/Android.bp"}} 1728func findBlueprintDescendants(paths []string) (descendants map[string][]string, err error) { 1729 // make mapping from dir path to file path 1730 filesByDir := make(map[string]string, len(paths)) 1731 for _, path := range paths { 1732 dir := filepath.Dir(path) 1733 _, alreadyFound := filesByDir[dir] 1734 if alreadyFound { 1735 return nil, fmt.Errorf("Found two Blueprint files in directory %v : %v and %v", dir, filesByDir[dir], path) 1736 } 1737 filesByDir[dir] = path 1738 } 1739 1740 findAncestor := func(childFile string) (ancestor string) { 1741 prevAncestorDir := filepath.Dir(childFile) 1742 for { 1743 ancestorDir := filepath.Dir(prevAncestorDir) 1744 if ancestorDir == prevAncestorDir { 1745 // reached the root dir without any matches; assign this as a descendant of "" 1746 return "" 1747 } 1748 1749 ancestorFile, ancestorExists := filesByDir[ancestorDir] 1750 if ancestorExists { 1751 return ancestorFile 1752 } 1753 prevAncestorDir = ancestorDir 1754 } 1755 } 1756 // generate the descendants map 1757 descendants = make(map[string][]string, len(filesByDir)) 1758 for _, childFile := range filesByDir { 1759 ancestorFile := findAncestor(childFile) 1760 descendants[ancestorFile] = append(descendants[ancestorFile], childFile) 1761 } 1762 return descendants, nil 1763} 1764 1765type visitOrderer interface { 1766 // returns the number of modules that this module needs to wait for 1767 waitCount(module *moduleInfo) int 1768 // returns the list of modules that are waiting for this module 1769 propagate(module *moduleInfo) []*moduleInfo 1770 // visit modules in order 1771 visit(modules []*moduleInfo, visit func(*moduleInfo) bool) 1772} 1773 1774type unorderedVisitorImpl struct{} 1775 1776func (unorderedVisitorImpl) waitCount(module *moduleInfo) int { 1777 return 0 1778} 1779 1780func (unorderedVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1781 return nil 1782} 1783 1784func (unorderedVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1785 for _, module := range modules { 1786 if visit(module) { 1787 return 1788 } 1789 } 1790} 1791 1792type bottomUpVisitorImpl struct{} 1793 1794func (bottomUpVisitorImpl) waitCount(module *moduleInfo) int { 1795 return len(module.forwardDeps) 1796} 1797 1798func (bottomUpVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1799 return module.reverseDeps 1800} 1801 1802func (bottomUpVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1803 for _, module := range modules { 1804 if visit(module) { 1805 return 1806 } 1807 } 1808} 1809 1810type topDownVisitorImpl struct{} 1811 1812func (topDownVisitorImpl) waitCount(module *moduleInfo) int { 1813 return len(module.reverseDeps) 1814} 1815 1816func (topDownVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1817 return module.forwardDeps 1818} 1819 1820func (topDownVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1821 for i := 0; i < len(modules); i++ { 1822 module := modules[len(modules)-1-i] 1823 if visit(module) { 1824 return 1825 } 1826 } 1827} 1828 1829var ( 1830 bottomUpVisitor bottomUpVisitorImpl 1831 topDownVisitor topDownVisitorImpl 1832) 1833 1834// Calls visit on each module, guaranteeing that visit is not called on a module until visit on all 1835// of its dependencies has finished. 1836func (c *Context) parallelVisit(order visitOrderer, visit func(group *moduleInfo) bool) { 1837 doneCh := make(chan *moduleInfo) 1838 cancelCh := make(chan bool) 1839 count := 0 1840 cancel := false 1841 var backlog []*moduleInfo 1842 const limit = 1000 1843 1844 for _, module := range c.modulesSorted { 1845 module.waitingCount = order.waitCount(module) 1846 } 1847 1848 visitOne := func(module *moduleInfo) { 1849 if count < limit { 1850 count++ 1851 go func() { 1852 ret := visit(module) 1853 if ret { 1854 cancelCh <- true 1855 } 1856 doneCh <- module 1857 }() 1858 } else { 1859 backlog = append(backlog, module) 1860 } 1861 } 1862 1863 for _, module := range c.modulesSorted { 1864 if module.waitingCount == 0 { 1865 visitOne(module) 1866 } 1867 } 1868 1869 for count > 0 || len(backlog) > 0 { 1870 select { 1871 case <-cancelCh: 1872 cancel = true 1873 backlog = nil 1874 case doneModule := <-doneCh: 1875 count-- 1876 if !cancel { 1877 for count < limit && len(backlog) > 0 { 1878 toVisit := backlog[0] 1879 backlog = backlog[1:] 1880 visitOne(toVisit) 1881 } 1882 for _, module := range order.propagate(doneModule) { 1883 module.waitingCount-- 1884 if module.waitingCount == 0 { 1885 visitOne(module) 1886 } 1887 } 1888 } 1889 } 1890 } 1891} 1892 1893// updateDependencies recursively walks the module dependency graph and updates 1894// additional fields based on the dependencies. It builds a sorted list of modules 1895// such that dependencies of a module always appear first, and populates reverse 1896// dependency links and counts of total dependencies. It also reports errors when 1897// it encounters dependency cycles. This should called after resolveDependencies, 1898// as well as after any mutator pass has called addDependency 1899func (c *Context) updateDependencies() (errs []error) { 1900 visited := make(map[*moduleInfo]bool) // modules that were already checked 1901 checking := make(map[*moduleInfo]bool) // modules actively being checked 1902 1903 sorted := make([]*moduleInfo, 0, len(c.moduleInfo)) 1904 1905 var check func(group *moduleInfo) []*moduleInfo 1906 1907 cycleError := func(cycle []*moduleInfo) { 1908 // We are the "start" of the cycle, so we're responsible 1909 // for generating the errors. The cycle list is in 1910 // reverse order because all the 'check' calls append 1911 // their own module to the list. 1912 errs = append(errs, &BlueprintError{ 1913 Err: fmt.Errorf("encountered dependency cycle:"), 1914 Pos: cycle[len(cycle)-1].pos, 1915 }) 1916 1917 // Iterate backwards through the cycle list. 1918 curModule := cycle[0] 1919 for i := len(cycle) - 1; i >= 0; i-- { 1920 nextModule := cycle[i] 1921 errs = append(errs, &BlueprintError{ 1922 Err: fmt.Errorf(" %q depends on %q", 1923 curModule.Name(), 1924 nextModule.Name()), 1925 Pos: curModule.pos, 1926 }) 1927 curModule = nextModule 1928 } 1929 } 1930 1931 check = func(module *moduleInfo) []*moduleInfo { 1932 visited[module] = true 1933 checking[module] = true 1934 defer delete(checking, module) 1935 1936 deps := make(map[*moduleInfo]bool) 1937 1938 // Add an implicit dependency ordering on all earlier modules in the same module group 1939 for _, dep := range module.group.modules { 1940 if dep == module { 1941 break 1942 } 1943 deps[dep] = true 1944 } 1945 1946 for _, dep := range module.directDeps { 1947 deps[dep.module] = true 1948 } 1949 1950 module.reverseDeps = []*moduleInfo{} 1951 module.forwardDeps = []*moduleInfo{} 1952 1953 for dep := range deps { 1954 if checking[dep] { 1955 // This is a cycle. 1956 return []*moduleInfo{dep, module} 1957 } 1958 1959 if !visited[dep] { 1960 cycle := check(dep) 1961 if cycle != nil { 1962 if cycle[0] == module { 1963 // We are the "start" of the cycle, so we're responsible 1964 // for generating the errors. The cycle list is in 1965 // reverse order because all the 'check' calls append 1966 // their own module to the list. 1967 cycleError(cycle) 1968 1969 // We can continue processing this module's children to 1970 // find more cycles. Since all the modules that were 1971 // part of the found cycle were marked as visited we 1972 // won't run into that cycle again. 1973 } else { 1974 // We're not the "start" of the cycle, so we just append 1975 // our module to the list and return it. 1976 return append(cycle, module) 1977 } 1978 } 1979 } 1980 1981 module.forwardDeps = append(module.forwardDeps, dep) 1982 dep.reverseDeps = append(dep.reverseDeps, module) 1983 } 1984 1985 sorted = append(sorted, module) 1986 1987 return nil 1988 } 1989 1990 for _, module := range c.moduleInfo { 1991 if !visited[module] { 1992 cycle := check(module) 1993 if cycle != nil { 1994 if cycle[len(cycle)-1] != module { 1995 panic("inconceivable!") 1996 } 1997 cycleError(cycle) 1998 } 1999 } 2000 } 2001 2002 c.modulesSorted = sorted 2003 2004 return 2005} 2006 2007// PrepareBuildActions generates an internal representation of all the build 2008// actions that need to be performed. This process involves invoking the 2009// GenerateBuildActions method on each of the Module objects created during the 2010// parse phase and then on each of the registered Singleton objects. 2011// 2012// If the ResolveDependencies method has not already been called it is called 2013// automatically by this method. 2014// 2015// The config argument is made available to all of the Module and Singleton 2016// objects via the Config method on the ModuleContext and SingletonContext 2017// objects passed to GenerateBuildActions. It is also passed to the functions 2018// specified via PoolFunc, RuleFunc, and VariableFunc so that they can compute 2019// config-specific values. 2020// 2021// The returned deps is a list of the ninja files dependencies that were added 2022// by the modules and singletons via the ModuleContext.AddNinjaFileDeps(), 2023// SingletonContext.AddNinjaFileDeps(), and PackageContext.AddNinjaFileDeps() 2024// methods. 2025func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) { 2026 pprof.Do(c.Context, pprof.Labels("blueprint", "PrepareBuildActions"), func(ctx context.Context) { 2027 c.buildActionsReady = false 2028 2029 if !c.dependenciesReady { 2030 var extraDeps []string 2031 extraDeps, errs = c.resolveDependencies(ctx, config) 2032 if len(errs) > 0 { 2033 return 2034 } 2035 deps = append(deps, extraDeps...) 2036 } 2037 2038 var depsModules []string 2039 depsModules, errs = c.generateModuleBuildActions(config, c.liveGlobals) 2040 if len(errs) > 0 { 2041 return 2042 } 2043 2044 var depsSingletons []string 2045 depsSingletons, errs = c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals) 2046 if len(errs) > 0 { 2047 return 2048 } 2049 2050 deps = append(deps, depsModules...) 2051 deps = append(deps, depsSingletons...) 2052 2053 if c.ninjaBuildDir != nil { 2054 err := c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir) 2055 if err != nil { 2056 errs = []error{err} 2057 return 2058 } 2059 } 2060 2061 pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals) 2062 2063 deps = append(deps, depsPackages...) 2064 2065 // This will panic if it finds a problem since it's a programming error. 2066 c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames) 2067 2068 c.pkgNames = pkgNames 2069 c.globalVariables = c.liveGlobals.variables 2070 c.globalPools = c.liveGlobals.pools 2071 c.globalRules = c.liveGlobals.rules 2072 2073 c.buildActionsReady = true 2074 }) 2075 2076 if len(errs) > 0 { 2077 return nil, errs 2078 } 2079 2080 return deps, nil 2081} 2082 2083func (c *Context) runMutators(ctx context.Context, config interface{}) (deps []string, errs []error) { 2084 var mutators []*mutatorInfo 2085 2086 pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) { 2087 mutators = append(mutators, c.earlyMutatorInfo...) 2088 mutators = append(mutators, c.mutatorInfo...) 2089 2090 for _, mutator := range mutators { 2091 pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) { 2092 var newDeps []string 2093 if mutator.topDownMutator != nil { 2094 newDeps, errs = c.runMutator(config, mutator, topDownMutator) 2095 } else if mutator.bottomUpMutator != nil { 2096 newDeps, errs = c.runMutator(config, mutator, bottomUpMutator) 2097 } else { 2098 panic("no mutator set on " + mutator.name) 2099 } 2100 if len(errs) > 0 { 2101 return 2102 } 2103 deps = append(deps, newDeps...) 2104 }) 2105 if len(errs) > 0 { 2106 return 2107 } 2108 } 2109 }) 2110 2111 if len(errs) > 0 { 2112 return nil, errs 2113 } 2114 2115 return deps, nil 2116} 2117 2118type mutatorDirection interface { 2119 run(mutator *mutatorInfo, ctx *mutatorContext) 2120 orderer() visitOrderer 2121 fmt.Stringer 2122} 2123 2124type bottomUpMutatorImpl struct{} 2125 2126func (bottomUpMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 2127 mutator.bottomUpMutator(ctx) 2128} 2129 2130func (bottomUpMutatorImpl) orderer() visitOrderer { 2131 return bottomUpVisitor 2132} 2133 2134func (bottomUpMutatorImpl) String() string { 2135 return "bottom up mutator" 2136} 2137 2138type topDownMutatorImpl struct{} 2139 2140func (topDownMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 2141 mutator.topDownMutator(ctx) 2142} 2143 2144func (topDownMutatorImpl) orderer() visitOrderer { 2145 return topDownVisitor 2146} 2147 2148func (topDownMutatorImpl) String() string { 2149 return "top down mutator" 2150} 2151 2152var ( 2153 topDownMutator topDownMutatorImpl 2154 bottomUpMutator bottomUpMutatorImpl 2155) 2156 2157type reverseDep struct { 2158 module *moduleInfo 2159 dep depInfo 2160} 2161 2162func (c *Context) runMutator(config interface{}, mutator *mutatorInfo, 2163 direction mutatorDirection) (deps []string, errs []error) { 2164 2165 newModuleInfo := make(map[Module]*moduleInfo) 2166 for k, v := range c.moduleInfo { 2167 newModuleInfo[k] = v 2168 } 2169 2170 type globalStateChange struct { 2171 reverse []reverseDep 2172 rename []rename 2173 replace []replace 2174 newModules []*moduleInfo 2175 deps []string 2176 } 2177 2178 reverseDeps := make(map[*moduleInfo][]depInfo) 2179 var rename []rename 2180 var replace []replace 2181 var newModules []*moduleInfo 2182 2183 errsCh := make(chan []error) 2184 globalStateCh := make(chan globalStateChange) 2185 newVariationsCh := make(chan []*moduleInfo) 2186 done := make(chan bool) 2187 2188 c.depsModified = 0 2189 2190 visit := func(module *moduleInfo) bool { 2191 if module.splitModules != nil { 2192 panic("split module found in sorted module list") 2193 } 2194 2195 mctx := &mutatorContext{ 2196 baseModuleContext: baseModuleContext{ 2197 context: c, 2198 config: config, 2199 module: module, 2200 }, 2201 name: mutator.name, 2202 } 2203 2204 func() { 2205 defer func() { 2206 if r := recover(); r != nil { 2207 in := fmt.Sprintf("%s %q for %s", direction, mutator.name, module) 2208 if err, ok := r.(panicError); ok { 2209 err.addIn(in) 2210 mctx.error(err) 2211 } else { 2212 mctx.error(newPanicErrorf(r, in)) 2213 } 2214 } 2215 }() 2216 direction.run(mutator, mctx) 2217 }() 2218 2219 if len(mctx.errs) > 0 { 2220 errsCh <- mctx.errs 2221 return true 2222 } 2223 2224 if len(mctx.newVariations) > 0 { 2225 newVariationsCh <- mctx.newVariations 2226 } 2227 2228 if len(mctx.reverseDeps) > 0 || len(mctx.replace) > 0 || len(mctx.rename) > 0 || len(mctx.newModules) > 0 || len(mctx.ninjaFileDeps) > 0 { 2229 globalStateCh <- globalStateChange{ 2230 reverse: mctx.reverseDeps, 2231 replace: mctx.replace, 2232 rename: mctx.rename, 2233 newModules: mctx.newModules, 2234 deps: mctx.ninjaFileDeps, 2235 } 2236 } 2237 2238 return false 2239 } 2240 2241 // Process errs and reverseDeps in a single goroutine 2242 go func() { 2243 for { 2244 select { 2245 case newErrs := <-errsCh: 2246 errs = append(errs, newErrs...) 2247 case globalStateChange := <-globalStateCh: 2248 for _, r := range globalStateChange.reverse { 2249 reverseDeps[r.module] = append(reverseDeps[r.module], r.dep) 2250 } 2251 replace = append(replace, globalStateChange.replace...) 2252 rename = append(rename, globalStateChange.rename...) 2253 newModules = append(newModules, globalStateChange.newModules...) 2254 deps = append(deps, globalStateChange.deps...) 2255 case newVariations := <-newVariationsCh: 2256 for _, m := range newVariations { 2257 newModuleInfo[m.logicModule] = m 2258 } 2259 case <-done: 2260 return 2261 } 2262 } 2263 }() 2264 2265 if mutator.parallel { 2266 c.parallelVisit(direction.orderer(), visit) 2267 } else { 2268 direction.orderer().visit(c.modulesSorted, visit) 2269 } 2270 2271 done <- true 2272 2273 if len(errs) > 0 { 2274 return nil, errs 2275 } 2276 2277 c.moduleInfo = newModuleInfo 2278 2279 for _, group := range c.moduleGroups { 2280 for i := 0; i < len(group.modules); i++ { 2281 module := group.modules[i] 2282 2283 // Update module group to contain newly split variants 2284 if module.splitModules != nil { 2285 group.modules, i = spliceModules(group.modules, i, module.splitModules) 2286 } 2287 2288 // Create any new aliases. 2289 if module.aliasTarget != nil { 2290 group.aliases = append(group.aliases, &moduleAlias{ 2291 variantName: module.variantName, 2292 variant: module.variant, 2293 dependencyVariant: module.dependencyVariant, 2294 target: module.aliasTarget, 2295 }) 2296 } 2297 2298 // Fix up any remaining dependencies on modules that were split into variants 2299 // by replacing them with the first variant 2300 for j, dep := range module.directDeps { 2301 if dep.module.logicModule == nil { 2302 module.directDeps[j].module = dep.module.splitModules[0] 2303 } 2304 } 2305 2306 if module.createdBy != nil && module.createdBy.logicModule == nil { 2307 module.createdBy = module.createdBy.splitModules[0] 2308 } 2309 2310 // Add in any new direct dependencies that were added by the mutator 2311 module.directDeps = append(module.directDeps, module.newDirectDeps...) 2312 module.newDirectDeps = nil 2313 } 2314 2315 // Forward or delete any dangling aliases. 2316 for i := 0; i < len(group.aliases); i++ { 2317 alias := group.aliases[i] 2318 2319 if alias.target.logicModule == nil { 2320 if alias.target.aliasTarget != nil { 2321 alias.target = alias.target.aliasTarget 2322 } else { 2323 // The alias was left dangling, remove it. 2324 group.aliases = append(group.aliases[:i], group.aliases[i+1:]...) 2325 i-- 2326 } 2327 } 2328 } 2329 } 2330 2331 // Add in any new reverse dependencies that were added by the mutator 2332 for module, deps := range reverseDeps { 2333 sort.Sort(depSorter(deps)) 2334 module.directDeps = append(module.directDeps, deps...) 2335 c.depsModified++ 2336 } 2337 2338 for _, module := range newModules { 2339 errs = c.addModule(module) 2340 if len(errs) > 0 { 2341 return nil, errs 2342 } 2343 atomic.AddUint32(&c.depsModified, 1) 2344 } 2345 2346 errs = c.handleRenames(rename) 2347 if len(errs) > 0 { 2348 return nil, errs 2349 } 2350 2351 errs = c.handleReplacements(replace) 2352 if len(errs) > 0 { 2353 return nil, errs 2354 } 2355 2356 if c.depsModified > 0 { 2357 errs = c.updateDependencies() 2358 if len(errs) > 0 { 2359 return nil, errs 2360 } 2361 } 2362 2363 return deps, errs 2364} 2365 2366// Replaces every build logic module with a clone of itself. Prevents introducing problems where 2367// a mutator sets a non-property member variable on a module, which works until a later mutator 2368// creates variants of that module. 2369func (c *Context) cloneModules() { 2370 type update struct { 2371 orig Module 2372 clone *moduleInfo 2373 } 2374 ch := make(chan update) 2375 doneCh := make(chan bool) 2376 go func() { 2377 c.parallelVisit(unorderedVisitorImpl{}, func(m *moduleInfo) bool { 2378 origLogicModule := m.logicModule 2379 m.logicModule, m.properties = c.cloneLogicModule(m) 2380 ch <- update{origLogicModule, m} 2381 return false 2382 }) 2383 doneCh <- true 2384 }() 2385 2386 done := false 2387 for !done { 2388 select { 2389 case <-doneCh: 2390 done = true 2391 case update := <-ch: 2392 delete(c.moduleInfo, update.orig) 2393 c.moduleInfo[update.clone.logicModule] = update.clone 2394 } 2395 } 2396} 2397 2398// Removes modules[i] from the list and inserts newModules... where it was located, returning 2399// the new slice and the index of the last inserted element 2400func spliceModules(modules []*moduleInfo, i int, newModules []*moduleInfo) ([]*moduleInfo, int) { 2401 spliceSize := len(newModules) 2402 newLen := len(modules) + spliceSize - 1 2403 var dest []*moduleInfo 2404 if cap(modules) >= len(modules)-1+len(newModules) { 2405 // We can fit the splice in the existing capacity, do everything in place 2406 dest = modules[:newLen] 2407 } else { 2408 dest = make([]*moduleInfo, newLen) 2409 copy(dest, modules[:i]) 2410 } 2411 2412 // Move the end of the slice over by spliceSize-1 2413 copy(dest[i+spliceSize:], modules[i+1:]) 2414 2415 // Copy the new modules into the slice 2416 copy(dest[i:], newModules) 2417 2418 return dest, i + spliceSize - 1 2419} 2420 2421func (c *Context) generateModuleBuildActions(config interface{}, 2422 liveGlobals *liveTracker) ([]string, []error) { 2423 2424 var deps []string 2425 var errs []error 2426 2427 cancelCh := make(chan struct{}) 2428 errsCh := make(chan []error) 2429 depsCh := make(chan []string) 2430 2431 go func() { 2432 for { 2433 select { 2434 case <-cancelCh: 2435 close(cancelCh) 2436 return 2437 case newErrs := <-errsCh: 2438 errs = append(errs, newErrs...) 2439 case newDeps := <-depsCh: 2440 deps = append(deps, newDeps...) 2441 2442 } 2443 } 2444 }() 2445 2446 c.parallelVisit(bottomUpVisitor, func(module *moduleInfo) bool { 2447 2448 uniqueName := c.nameInterface.UniqueName(newNamespaceContext(module), module.group.name) 2449 sanitizedName := toNinjaName(uniqueName) 2450 2451 prefix := moduleNamespacePrefix(sanitizedName + "_" + module.variantName) 2452 2453 // The parent scope of the moduleContext's local scope gets overridden to be that of the 2454 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2455 // just set it to nil. 2456 scope := newLocalScope(nil, prefix) 2457 2458 mctx := &moduleContext{ 2459 baseModuleContext: baseModuleContext{ 2460 context: c, 2461 config: config, 2462 module: module, 2463 }, 2464 scope: scope, 2465 handledMissingDeps: module.missingDeps == nil, 2466 } 2467 2468 func() { 2469 defer func() { 2470 if r := recover(); r != nil { 2471 in := fmt.Sprintf("GenerateBuildActions for %s", module) 2472 if err, ok := r.(panicError); ok { 2473 err.addIn(in) 2474 mctx.error(err) 2475 } else { 2476 mctx.error(newPanicErrorf(r, in)) 2477 } 2478 } 2479 }() 2480 mctx.module.logicModule.GenerateBuildActions(mctx) 2481 }() 2482 2483 if len(mctx.errs) > 0 { 2484 errsCh <- mctx.errs 2485 return true 2486 } 2487 2488 if module.missingDeps != nil && !mctx.handledMissingDeps { 2489 var errs []error 2490 for _, depName := range module.missingDeps { 2491 errs = append(errs, c.missingDependencyError(module, depName)) 2492 } 2493 errsCh <- errs 2494 return true 2495 } 2496 2497 depsCh <- mctx.ninjaFileDeps 2498 2499 newErrs := c.processLocalBuildActions(&module.actionDefs, 2500 &mctx.actionDefs, liveGlobals) 2501 if len(newErrs) > 0 { 2502 errsCh <- newErrs 2503 return true 2504 } 2505 return false 2506 }) 2507 2508 cancelCh <- struct{}{} 2509 <-cancelCh 2510 2511 return deps, errs 2512} 2513 2514func (c *Context) generateSingletonBuildActions(config interface{}, 2515 singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { 2516 2517 var deps []string 2518 var errs []error 2519 2520 for _, info := range singletons { 2521 // The parent scope of the singletonContext's local scope gets overridden to be that of the 2522 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2523 // just set it to nil. 2524 scope := newLocalScope(nil, singletonNamespacePrefix(info.name)) 2525 2526 sctx := &singletonContext{ 2527 name: info.name, 2528 context: c, 2529 config: config, 2530 scope: scope, 2531 globals: liveGlobals, 2532 } 2533 2534 func() { 2535 defer func() { 2536 if r := recover(); r != nil { 2537 in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name) 2538 if err, ok := r.(panicError); ok { 2539 err.addIn(in) 2540 sctx.error(err) 2541 } else { 2542 sctx.error(newPanicErrorf(r, in)) 2543 } 2544 } 2545 }() 2546 info.singleton.GenerateBuildActions(sctx) 2547 }() 2548 2549 if len(sctx.errs) > 0 { 2550 errs = append(errs, sctx.errs...) 2551 if len(errs) > maxErrors { 2552 break 2553 } 2554 continue 2555 } 2556 2557 deps = append(deps, sctx.ninjaFileDeps...) 2558 2559 newErrs := c.processLocalBuildActions(&info.actionDefs, 2560 &sctx.actionDefs, liveGlobals) 2561 errs = append(errs, newErrs...) 2562 if len(errs) > maxErrors { 2563 break 2564 } 2565 } 2566 2567 return deps, errs 2568} 2569 2570func (c *Context) processLocalBuildActions(out, in *localBuildActions, 2571 liveGlobals *liveTracker) []error { 2572 2573 var errs []error 2574 2575 // First we go through and add everything referenced by the module's 2576 // buildDefs to the live globals set. This will end up adding the live 2577 // locals to the set as well, but we'll take them out after. 2578 for _, def := range in.buildDefs { 2579 err := liveGlobals.AddBuildDefDeps(def) 2580 if err != nil { 2581 errs = append(errs, err) 2582 } 2583 } 2584 2585 if len(errs) > 0 { 2586 return errs 2587 } 2588 2589 out.buildDefs = append(out.buildDefs, in.buildDefs...) 2590 2591 // We use the now-incorrect set of live "globals" to determine which local 2592 // definitions are live. As we go through copying those live locals to the 2593 // moduleGroup we remove them from the live globals set. 2594 for _, v := range in.variables { 2595 isLive := liveGlobals.RemoveVariableIfLive(v) 2596 if isLive { 2597 out.variables = append(out.variables, v) 2598 } 2599 } 2600 2601 for _, r := range in.rules { 2602 isLive := liveGlobals.RemoveRuleIfLive(r) 2603 if isLive { 2604 out.rules = append(out.rules, r) 2605 } 2606 } 2607 2608 return nil 2609} 2610 2611func (c *Context) walkDeps(topModule *moduleInfo, allowDuplicates bool, 2612 visitDown func(depInfo, *moduleInfo) bool, visitUp func(depInfo, *moduleInfo)) { 2613 2614 visited := make(map[*moduleInfo]bool) 2615 var visiting *moduleInfo 2616 2617 defer func() { 2618 if r := recover(); r != nil { 2619 panic(newPanicErrorf(r, "WalkDeps(%s, %s, %s) for dependency %s", 2620 topModule, funcName(visitDown), funcName(visitUp), visiting)) 2621 } 2622 }() 2623 2624 var walk func(module *moduleInfo) 2625 walk = func(module *moduleInfo) { 2626 for _, dep := range module.directDeps { 2627 if allowDuplicates || !visited[dep.module] { 2628 visiting = dep.module 2629 recurse := true 2630 if visitDown != nil { 2631 recurse = visitDown(dep, module) 2632 } 2633 if recurse && !visited[dep.module] { 2634 walk(dep.module) 2635 visited[dep.module] = true 2636 } 2637 if visitUp != nil { 2638 visitUp(dep, module) 2639 } 2640 } 2641 } 2642 } 2643 2644 walk(topModule) 2645} 2646 2647type replace struct { 2648 from, to *moduleInfo 2649} 2650 2651type rename struct { 2652 group *moduleGroup 2653 name string 2654} 2655 2656func (c *Context) moduleMatchingVariant(module *moduleInfo, name string) *moduleInfo { 2657 group := c.moduleGroupFromName(name, module.namespace()) 2658 2659 if group == nil { 2660 return nil 2661 } 2662 2663 for _, m := range group.modules { 2664 if module.variantName == m.variantName { 2665 return m 2666 } 2667 } 2668 2669 for _, m := range group.aliases { 2670 if module.variantName == m.variantName { 2671 return m.target 2672 } 2673 } 2674 2675 return nil 2676} 2677 2678func (c *Context) handleRenames(renames []rename) []error { 2679 var errs []error 2680 for _, rename := range renames { 2681 group, name := rename.group, rename.name 2682 if name == group.name || len(group.modules) < 1 { 2683 continue 2684 } 2685 2686 errs = append(errs, c.nameInterface.Rename(group.name, rename.name, group.namespace)...) 2687 } 2688 2689 return errs 2690} 2691 2692func (c *Context) handleReplacements(replacements []replace) []error { 2693 var errs []error 2694 for _, replace := range replacements { 2695 for _, m := range replace.from.reverseDeps { 2696 for i, d := range m.directDeps { 2697 if d.module == replace.from { 2698 m.directDeps[i].module = replace.to 2699 } 2700 } 2701 } 2702 2703 atomic.AddUint32(&c.depsModified, 1) 2704 } 2705 2706 return errs 2707} 2708 2709func (c *Context) discoveredMissingDependencies(module *moduleInfo, depName string) (errs []error) { 2710 if c.allowMissingDependencies { 2711 module.missingDeps = append(module.missingDeps, depName) 2712 return nil 2713 } 2714 return []error{c.missingDependencyError(module, depName)} 2715} 2716 2717func (c *Context) missingDependencyError(module *moduleInfo, depName string) (errs error) { 2718 err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName) 2719 2720 return &BlueprintError{ 2721 Err: err, 2722 Pos: module.pos, 2723 } 2724} 2725 2726func (c *Context) moduleGroupFromName(name string, namespace Namespace) *moduleGroup { 2727 group, exists := c.nameInterface.ModuleFromName(name, namespace) 2728 if exists { 2729 return group.moduleGroup 2730 } 2731 return nil 2732} 2733 2734func (c *Context) sortedModuleGroups() []*moduleGroup { 2735 if c.cachedSortedModuleGroups == nil { 2736 unwrap := func(wrappers []ModuleGroup) []*moduleGroup { 2737 result := make([]*moduleGroup, 0, len(wrappers)) 2738 for _, group := range wrappers { 2739 result = append(result, group.moduleGroup) 2740 } 2741 return result 2742 } 2743 2744 c.cachedSortedModuleGroups = unwrap(c.nameInterface.AllModules()) 2745 } 2746 2747 return c.cachedSortedModuleGroups 2748} 2749 2750func (c *Context) visitAllModules(visit func(Module)) { 2751 var module *moduleInfo 2752 2753 defer func() { 2754 if r := recover(); r != nil { 2755 panic(newPanicErrorf(r, "VisitAllModules(%s) for %s", 2756 funcName(visit), module)) 2757 } 2758 }() 2759 2760 for _, moduleGroup := range c.sortedModuleGroups() { 2761 for _, module = range moduleGroup.modules { 2762 visit(module.logicModule) 2763 } 2764 } 2765} 2766 2767func (c *Context) visitAllModulesIf(pred func(Module) bool, 2768 visit func(Module)) { 2769 2770 var module *moduleInfo 2771 2772 defer func() { 2773 if r := recover(); r != nil { 2774 panic(newPanicErrorf(r, "VisitAllModulesIf(%s, %s) for %s", 2775 funcName(pred), funcName(visit), module)) 2776 } 2777 }() 2778 2779 for _, moduleGroup := range c.sortedModuleGroups() { 2780 for _, module := range moduleGroup.modules { 2781 if pred(module.logicModule) { 2782 visit(module.logicModule) 2783 } 2784 } 2785 } 2786} 2787 2788func (c *Context) visitAllModuleVariants(module *moduleInfo, 2789 visit func(Module)) { 2790 2791 var variant *moduleInfo 2792 2793 defer func() { 2794 if r := recover(); r != nil { 2795 panic(newPanicErrorf(r, "VisitAllModuleVariants(%s, %s) for %s", 2796 module, funcName(visit), variant)) 2797 } 2798 }() 2799 2800 for _, variant = range module.group.modules { 2801 visit(variant.logicModule) 2802 } 2803} 2804 2805func (c *Context) requireNinjaVersion(major, minor, micro int) { 2806 if major != 1 { 2807 panic("ninja version with major version != 1 not supported") 2808 } 2809 if c.requiredNinjaMinor < minor { 2810 c.requiredNinjaMinor = minor 2811 c.requiredNinjaMicro = micro 2812 } 2813 if c.requiredNinjaMinor == minor && c.requiredNinjaMicro < micro { 2814 c.requiredNinjaMicro = micro 2815 } 2816} 2817 2818func (c *Context) setNinjaBuildDir(value ninjaString) { 2819 if c.ninjaBuildDir == nil { 2820 c.ninjaBuildDir = value 2821 } 2822} 2823 2824func (c *Context) makeUniquePackageNames( 2825 liveGlobals *liveTracker) (map[*packageContext]string, []string) { 2826 2827 pkgs := make(map[string]*packageContext) 2828 pkgNames := make(map[*packageContext]string) 2829 longPkgNames := make(map[*packageContext]bool) 2830 2831 processPackage := func(pctx *packageContext) { 2832 if pctx == nil { 2833 // This is a built-in rule and has no package. 2834 return 2835 } 2836 if _, ok := pkgNames[pctx]; ok { 2837 // We've already processed this package. 2838 return 2839 } 2840 2841 otherPkg, present := pkgs[pctx.shortName] 2842 if present { 2843 // Short name collision. Both this package and the one that's 2844 // already there need to use their full names. We leave the short 2845 // name in pkgNames for now so future collisions still get caught. 2846 longPkgNames[pctx] = true 2847 longPkgNames[otherPkg] = true 2848 } else { 2849 // No collision so far. Tentatively set the package's name to be 2850 // its short name. 2851 pkgNames[pctx] = pctx.shortName 2852 pkgs[pctx.shortName] = pctx 2853 } 2854 } 2855 2856 // We try to give all packages their short name, but when we get collisions 2857 // we need to use the full unique package name. 2858 for v, _ := range liveGlobals.variables { 2859 processPackage(v.packageContext()) 2860 } 2861 for p, _ := range liveGlobals.pools { 2862 processPackage(p.packageContext()) 2863 } 2864 for r, _ := range liveGlobals.rules { 2865 processPackage(r.packageContext()) 2866 } 2867 2868 // Add the packages that had collisions using their full unique names. This 2869 // will overwrite any short names that were added in the previous step. 2870 for pctx := range longPkgNames { 2871 pkgNames[pctx] = pctx.fullName 2872 } 2873 2874 // Create deps list from calls to PackageContext.AddNinjaFileDeps 2875 deps := []string{} 2876 for _, pkg := range pkgs { 2877 deps = append(deps, pkg.ninjaFileDeps...) 2878 } 2879 2880 return pkgNames, deps 2881} 2882 2883func (c *Context) checkForVariableReferenceCycles( 2884 variables map[Variable]ninjaString, pkgNames map[*packageContext]string) { 2885 2886 visited := make(map[Variable]bool) // variables that were already checked 2887 checking := make(map[Variable]bool) // variables actively being checked 2888 2889 var check func(v Variable) []Variable 2890 2891 check = func(v Variable) []Variable { 2892 visited[v] = true 2893 checking[v] = true 2894 defer delete(checking, v) 2895 2896 value := variables[v] 2897 for _, dep := range value.Variables() { 2898 if checking[dep] { 2899 // This is a cycle. 2900 return []Variable{dep, v} 2901 } 2902 2903 if !visited[dep] { 2904 cycle := check(dep) 2905 if cycle != nil { 2906 if cycle[0] == v { 2907 // We are the "start" of the cycle, so we're responsible 2908 // for generating the errors. The cycle list is in 2909 // reverse order because all the 'check' calls append 2910 // their own module to the list. 2911 msgs := []string{"detected variable reference cycle:"} 2912 2913 // Iterate backwards through the cycle list. 2914 curName := v.fullName(pkgNames) 2915 curValue := value.Value(pkgNames) 2916 for i := len(cycle) - 1; i >= 0; i-- { 2917 next := cycle[i] 2918 nextName := next.fullName(pkgNames) 2919 nextValue := variables[next].Value(pkgNames) 2920 2921 msgs = append(msgs, fmt.Sprintf( 2922 " %q depends on %q", curName, nextName)) 2923 msgs = append(msgs, fmt.Sprintf( 2924 " [%s = %s]", curName, curValue)) 2925 2926 curName = nextName 2927 curValue = nextValue 2928 } 2929 2930 // Variable reference cycles are a programming error, 2931 // not the fault of the Blueprint file authors. 2932 panic(strings.Join(msgs, "\n")) 2933 } else { 2934 // We're not the "start" of the cycle, so we just append 2935 // our module to the list and return it. 2936 return append(cycle, v) 2937 } 2938 } 2939 } 2940 } 2941 2942 return nil 2943 } 2944 2945 for v := range variables { 2946 if !visited[v] { 2947 cycle := check(v) 2948 if cycle != nil { 2949 panic("inconceivable!") 2950 } 2951 } 2952 } 2953} 2954 2955// AllTargets returns a map all the build target names to the rule used to build 2956// them. This is the same information that is output by running 'ninja -t 2957// targets all'. If this is called before PrepareBuildActions successfully 2958// completes then ErrbuildActionsNotReady is returned. 2959func (c *Context) AllTargets() (map[string]string, error) { 2960 if !c.buildActionsReady { 2961 return nil, ErrBuildActionsNotReady 2962 } 2963 2964 targets := map[string]string{} 2965 2966 // Collect all the module build targets. 2967 for _, module := range c.moduleInfo { 2968 for _, buildDef := range module.actionDefs.buildDefs { 2969 ruleName := buildDef.Rule.fullName(c.pkgNames) 2970 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2971 outputValue, err := output.Eval(c.globalVariables) 2972 if err != nil { 2973 return nil, err 2974 } 2975 targets[outputValue] = ruleName 2976 } 2977 } 2978 } 2979 2980 // Collect all the singleton build targets. 2981 for _, info := range c.singletonInfo { 2982 for _, buildDef := range info.actionDefs.buildDefs { 2983 ruleName := buildDef.Rule.fullName(c.pkgNames) 2984 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2985 outputValue, err := output.Eval(c.globalVariables) 2986 if err != nil { 2987 return nil, err 2988 } 2989 targets[outputValue] = ruleName 2990 } 2991 } 2992 } 2993 2994 return targets, nil 2995} 2996 2997func (c *Context) NinjaBuildDir() (string, error) { 2998 if c.ninjaBuildDir != nil { 2999 return c.ninjaBuildDir.Eval(c.globalVariables) 3000 } else { 3001 return "", nil 3002 } 3003} 3004 3005// ModuleTypePropertyStructs returns a mapping from module type name to a list of pointers to 3006// property structs returned by the factory for that module type. 3007func (c *Context) ModuleTypePropertyStructs() map[string][]interface{} { 3008 ret := make(map[string][]interface{}) 3009 for moduleType, factory := range c.moduleFactories { 3010 _, ret[moduleType] = factory() 3011 } 3012 3013 return ret 3014} 3015 3016func (c *Context) ModuleTypeFactories() map[string]ModuleFactory { 3017 ret := make(map[string]ModuleFactory) 3018 for k, v := range c.moduleFactories { 3019 ret[k] = v 3020 } 3021 return ret 3022} 3023 3024func (c *Context) ModuleName(logicModule Module) string { 3025 module := c.moduleInfo[logicModule] 3026 return module.Name() 3027} 3028 3029func (c *Context) ModulePath(logicModule Module) string { 3030 module := c.moduleInfo[logicModule] 3031 return module.relBlueprintsFile 3032} 3033 3034func (c *Context) ModuleDir(logicModule Module) string { 3035 return filepath.Dir(c.ModulePath(logicModule)) 3036} 3037 3038func (c *Context) ModuleSubDir(logicModule Module) string { 3039 module := c.moduleInfo[logicModule] 3040 return module.variantName 3041} 3042 3043func (c *Context) ModuleType(logicModule Module) string { 3044 module := c.moduleInfo[logicModule] 3045 return module.typeName 3046} 3047 3048func (c *Context) BlueprintFile(logicModule Module) string { 3049 module := c.moduleInfo[logicModule] 3050 return module.relBlueprintsFile 3051} 3052 3053func (c *Context) ModuleErrorf(logicModule Module, format string, 3054 args ...interface{}) error { 3055 3056 module := c.moduleInfo[logicModule] 3057 return &BlueprintError{ 3058 Err: fmt.Errorf(format, args...), 3059 Pos: module.pos, 3060 } 3061} 3062 3063func (c *Context) VisitAllModules(visit func(Module)) { 3064 c.visitAllModules(visit) 3065} 3066 3067func (c *Context) VisitAllModulesIf(pred func(Module) bool, 3068 visit func(Module)) { 3069 3070 c.visitAllModulesIf(pred, visit) 3071} 3072 3073func (c *Context) VisitDirectDeps(module Module, visit func(Module)) { 3074 topModule := c.moduleInfo[module] 3075 3076 var visiting *moduleInfo 3077 3078 defer func() { 3079 if r := recover(); r != nil { 3080 panic(newPanicErrorf(r, "VisitDirectDeps(%s, %s) for dependency %s", 3081 topModule, funcName(visit), visiting)) 3082 } 3083 }() 3084 3085 for _, dep := range topModule.directDeps { 3086 visiting = dep.module 3087 visit(dep.module.logicModule) 3088 } 3089} 3090 3091func (c *Context) VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) { 3092 topModule := c.moduleInfo[module] 3093 3094 var visiting *moduleInfo 3095 3096 defer func() { 3097 if r := recover(); r != nil { 3098 panic(newPanicErrorf(r, "VisitDirectDepsIf(%s, %s, %s) for dependency %s", 3099 topModule, funcName(pred), funcName(visit), visiting)) 3100 } 3101 }() 3102 3103 for _, dep := range topModule.directDeps { 3104 visiting = dep.module 3105 if pred(dep.module.logicModule) { 3106 visit(dep.module.logicModule) 3107 } 3108 } 3109} 3110 3111func (c *Context) VisitDepsDepthFirst(module Module, visit func(Module)) { 3112 topModule := c.moduleInfo[module] 3113 3114 var visiting *moduleInfo 3115 3116 defer func() { 3117 if r := recover(); r != nil { 3118 panic(newPanicErrorf(r, "VisitDepsDepthFirst(%s, %s) for dependency %s", 3119 topModule, funcName(visit), visiting)) 3120 } 3121 }() 3122 3123 c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) { 3124 visiting = dep.module 3125 visit(dep.module.logicModule) 3126 }) 3127} 3128 3129func (c *Context) VisitDepsDepthFirstIf(module Module, pred func(Module) bool, visit func(Module)) { 3130 topModule := c.moduleInfo[module] 3131 3132 var visiting *moduleInfo 3133 3134 defer func() { 3135 if r := recover(); r != nil { 3136 panic(newPanicErrorf(r, "VisitDepsDepthFirstIf(%s, %s, %s) for dependency %s", 3137 topModule, funcName(pred), funcName(visit), visiting)) 3138 } 3139 }() 3140 3141 c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) { 3142 if pred(dep.module.logicModule) { 3143 visiting = dep.module 3144 visit(dep.module.logicModule) 3145 } 3146 }) 3147} 3148 3149func (c *Context) PrimaryModule(module Module) Module { 3150 return c.moduleInfo[module].group.modules[0].logicModule 3151} 3152 3153func (c *Context) FinalModule(module Module) Module { 3154 modules := c.moduleInfo[module].group.modules 3155 return modules[len(modules)-1].logicModule 3156} 3157 3158func (c *Context) VisitAllModuleVariants(module Module, 3159 visit func(Module)) { 3160 3161 c.visitAllModuleVariants(c.moduleInfo[module], visit) 3162} 3163 3164// Singletons returns a list of all registered Singletons. 3165func (c *Context) Singletons() []Singleton { 3166 var ret []Singleton 3167 for _, s := range c.singletonInfo { 3168 ret = append(ret, s.singleton) 3169 } 3170 return ret 3171} 3172 3173// SingletonName returns the name that the given singleton was registered with. 3174func (c *Context) SingletonName(singleton Singleton) string { 3175 for _, s := range c.singletonInfo { 3176 if s.singleton == singleton { 3177 return s.name 3178 } 3179 } 3180 return "" 3181} 3182 3183// WriteBuildFile writes the Ninja manifeset text for the generated build 3184// actions to w. If this is called before PrepareBuildActions successfully 3185// completes then ErrBuildActionsNotReady is returned. 3186func (c *Context) WriteBuildFile(w io.Writer) error { 3187 var err error 3188 pprof.Do(c.Context, pprof.Labels("blueprint", "WriteBuildFile"), func(ctx context.Context) { 3189 if !c.buildActionsReady { 3190 err = ErrBuildActionsNotReady 3191 return 3192 } 3193 3194 nw := newNinjaWriter(w) 3195 3196 err = c.writeBuildFileHeader(nw) 3197 if err != nil { 3198 return 3199 } 3200 3201 err = c.writeNinjaRequiredVersion(nw) 3202 if err != nil { 3203 return 3204 } 3205 3206 err = c.writeSubninjas(nw) 3207 if err != nil { 3208 return 3209 } 3210 3211 // TODO: Group the globals by package. 3212 3213 err = c.writeGlobalVariables(nw) 3214 if err != nil { 3215 return 3216 } 3217 3218 err = c.writeGlobalPools(nw) 3219 if err != nil { 3220 return 3221 } 3222 3223 err = c.writeBuildDir(nw) 3224 if err != nil { 3225 return 3226 } 3227 3228 err = c.writeGlobalRules(nw) 3229 if err != nil { 3230 return 3231 } 3232 3233 err = c.writeAllModuleActions(nw) 3234 if err != nil { 3235 return 3236 } 3237 3238 err = c.writeAllSingletonActions(nw) 3239 if err != nil { 3240 return 3241 } 3242 }) 3243 3244 if err != nil { 3245 return err 3246 } 3247 3248 return nil 3249} 3250 3251type pkgAssociation struct { 3252 PkgName string 3253 PkgPath string 3254} 3255 3256type pkgAssociationSorter struct { 3257 pkgs []pkgAssociation 3258} 3259 3260func (s *pkgAssociationSorter) Len() int { 3261 return len(s.pkgs) 3262} 3263 3264func (s *pkgAssociationSorter) Less(i, j int) bool { 3265 iName := s.pkgs[i].PkgName 3266 jName := s.pkgs[j].PkgName 3267 return iName < jName 3268} 3269 3270func (s *pkgAssociationSorter) Swap(i, j int) { 3271 s.pkgs[i], s.pkgs[j] = s.pkgs[j], s.pkgs[i] 3272} 3273 3274func (c *Context) writeBuildFileHeader(nw *ninjaWriter) error { 3275 headerTemplate := template.New("fileHeader") 3276 _, err := headerTemplate.Parse(fileHeaderTemplate) 3277 if err != nil { 3278 // This is a programming error. 3279 panic(err) 3280 } 3281 3282 var pkgs []pkgAssociation 3283 maxNameLen := 0 3284 for pkg, name := range c.pkgNames { 3285 pkgs = append(pkgs, pkgAssociation{ 3286 PkgName: name, 3287 PkgPath: pkg.pkgPath, 3288 }) 3289 if len(name) > maxNameLen { 3290 maxNameLen = len(name) 3291 } 3292 } 3293 3294 for i := range pkgs { 3295 pkgs[i].PkgName += strings.Repeat(" ", maxNameLen-len(pkgs[i].PkgName)) 3296 } 3297 3298 sort.Sort(&pkgAssociationSorter{pkgs}) 3299 3300 params := map[string]interface{}{ 3301 "Pkgs": pkgs, 3302 } 3303 3304 buf := bytes.NewBuffer(nil) 3305 err = headerTemplate.Execute(buf, params) 3306 if err != nil { 3307 return err 3308 } 3309 3310 return nw.Comment(buf.String()) 3311} 3312 3313func (c *Context) writeNinjaRequiredVersion(nw *ninjaWriter) error { 3314 value := fmt.Sprintf("%d.%d.%d", c.requiredNinjaMajor, c.requiredNinjaMinor, 3315 c.requiredNinjaMicro) 3316 3317 err := nw.Assign("ninja_required_version", value) 3318 if err != nil { 3319 return err 3320 } 3321 3322 return nw.BlankLine() 3323} 3324 3325func (c *Context) writeSubninjas(nw *ninjaWriter) error { 3326 for _, subninja := range c.subninjas { 3327 err := nw.Subninja(subninja) 3328 if err != nil { 3329 return err 3330 } 3331 } 3332 return nw.BlankLine() 3333} 3334 3335func (c *Context) writeBuildDir(nw *ninjaWriter) error { 3336 if c.ninjaBuildDir != nil { 3337 err := nw.Assign("builddir", c.ninjaBuildDir.Value(c.pkgNames)) 3338 if err != nil { 3339 return err 3340 } 3341 3342 err = nw.BlankLine() 3343 if err != nil { 3344 return err 3345 } 3346 } 3347 return nil 3348} 3349 3350type globalEntity interface { 3351 fullName(pkgNames map[*packageContext]string) string 3352} 3353 3354type globalEntitySorter struct { 3355 pkgNames map[*packageContext]string 3356 entities []globalEntity 3357} 3358 3359func (s *globalEntitySorter) Len() int { 3360 return len(s.entities) 3361} 3362 3363func (s *globalEntitySorter) Less(i, j int) bool { 3364 iName := s.entities[i].fullName(s.pkgNames) 3365 jName := s.entities[j].fullName(s.pkgNames) 3366 return iName < jName 3367} 3368 3369func (s *globalEntitySorter) Swap(i, j int) { 3370 s.entities[i], s.entities[j] = s.entities[j], s.entities[i] 3371} 3372 3373func (c *Context) writeGlobalVariables(nw *ninjaWriter) error { 3374 visited := make(map[Variable]bool) 3375 3376 var walk func(v Variable) error 3377 walk = func(v Variable) error { 3378 visited[v] = true 3379 3380 // First visit variables on which this variable depends. 3381 value := c.globalVariables[v] 3382 for _, dep := range value.Variables() { 3383 if !visited[dep] { 3384 err := walk(dep) 3385 if err != nil { 3386 return err 3387 } 3388 } 3389 } 3390 3391 err := nw.Assign(v.fullName(c.pkgNames), value.Value(c.pkgNames)) 3392 if err != nil { 3393 return err 3394 } 3395 3396 err = nw.BlankLine() 3397 if err != nil { 3398 return err 3399 } 3400 3401 return nil 3402 } 3403 3404 globalVariables := make([]globalEntity, 0, len(c.globalVariables)) 3405 for variable := range c.globalVariables { 3406 globalVariables = append(globalVariables, variable) 3407 } 3408 3409 sort.Sort(&globalEntitySorter{c.pkgNames, globalVariables}) 3410 3411 for _, entity := range globalVariables { 3412 v := entity.(Variable) 3413 if !visited[v] { 3414 err := walk(v) 3415 if err != nil { 3416 return nil 3417 } 3418 } 3419 } 3420 3421 return nil 3422} 3423 3424func (c *Context) writeGlobalPools(nw *ninjaWriter) error { 3425 globalPools := make([]globalEntity, 0, len(c.globalPools)) 3426 for pool := range c.globalPools { 3427 globalPools = append(globalPools, pool) 3428 } 3429 3430 sort.Sort(&globalEntitySorter{c.pkgNames, globalPools}) 3431 3432 for _, entity := range globalPools { 3433 pool := entity.(Pool) 3434 name := pool.fullName(c.pkgNames) 3435 def := c.globalPools[pool] 3436 err := def.WriteTo(nw, name) 3437 if err != nil { 3438 return err 3439 } 3440 3441 err = nw.BlankLine() 3442 if err != nil { 3443 return err 3444 } 3445 } 3446 3447 return nil 3448} 3449 3450func (c *Context) writeGlobalRules(nw *ninjaWriter) error { 3451 globalRules := make([]globalEntity, 0, len(c.globalRules)) 3452 for rule := range c.globalRules { 3453 globalRules = append(globalRules, rule) 3454 } 3455 3456 sort.Sort(&globalEntitySorter{c.pkgNames, globalRules}) 3457 3458 for _, entity := range globalRules { 3459 rule := entity.(Rule) 3460 name := rule.fullName(c.pkgNames) 3461 def := c.globalRules[rule] 3462 err := def.WriteTo(nw, name, c.pkgNames) 3463 if err != nil { 3464 return err 3465 } 3466 3467 err = nw.BlankLine() 3468 if err != nil { 3469 return err 3470 } 3471 } 3472 3473 return nil 3474} 3475 3476type depSorter []depInfo 3477 3478func (s depSorter) Len() int { 3479 return len(s) 3480} 3481 3482func (s depSorter) Less(i, j int) bool { 3483 iName := s[i].module.Name() 3484 jName := s[j].module.Name() 3485 if iName == jName { 3486 iName = s[i].module.variantName 3487 jName = s[j].module.variantName 3488 } 3489 return iName < jName 3490} 3491 3492func (s depSorter) Swap(i, j int) { 3493 s[i], s[j] = s[j], s[i] 3494} 3495 3496type moduleSorter struct { 3497 modules []*moduleInfo 3498 nameInterface NameInterface 3499} 3500 3501func (s moduleSorter) Len() int { 3502 return len(s.modules) 3503} 3504 3505func (s moduleSorter) Less(i, j int) bool { 3506 iMod := s.modules[i] 3507 jMod := s.modules[j] 3508 iName := s.nameInterface.UniqueName(newNamespaceContext(iMod), iMod.group.name) 3509 jName := s.nameInterface.UniqueName(newNamespaceContext(jMod), jMod.group.name) 3510 if iName == jName { 3511 iName = s.modules[i].variantName 3512 jName = s.modules[j].variantName 3513 } 3514 3515 if iName == jName { 3516 panic(fmt.Sprintf("duplicate module name: %s: %#v and %#v\n", iName, iMod, jMod)) 3517 } 3518 return iName < jName 3519} 3520 3521func (s moduleSorter) Swap(i, j int) { 3522 s.modules[i], s.modules[j] = s.modules[j], s.modules[i] 3523} 3524 3525func (c *Context) writeAllModuleActions(nw *ninjaWriter) error { 3526 headerTemplate := template.New("moduleHeader") 3527 _, err := headerTemplate.Parse(moduleHeaderTemplate) 3528 if err != nil { 3529 // This is a programming error. 3530 panic(err) 3531 } 3532 3533 modules := make([]*moduleInfo, 0, len(c.moduleInfo)) 3534 for _, module := range c.moduleInfo { 3535 modules = append(modules, module) 3536 } 3537 sort.Sort(moduleSorter{modules, c.nameInterface}) 3538 3539 buf := bytes.NewBuffer(nil) 3540 3541 for _, module := range modules { 3542 if len(module.actionDefs.variables)+len(module.actionDefs.rules)+len(module.actionDefs.buildDefs) == 0 { 3543 continue 3544 } 3545 3546 buf.Reset() 3547 3548 // In order to make the bootstrap build manifest independent of the 3549 // build dir we need to output the Blueprints file locations in the 3550 // comments as paths relative to the source directory. 3551 relPos := module.pos 3552 relPos.Filename = module.relBlueprintsFile 3553 3554 // Get the name and location of the factory function for the module. 3555 factoryFunc := runtime.FuncForPC(reflect.ValueOf(module.factory).Pointer()) 3556 factoryName := factoryFunc.Name() 3557 3558 infoMap := map[string]interface{}{ 3559 "name": module.Name(), 3560 "typeName": module.typeName, 3561 "goFactory": factoryName, 3562 "pos": relPos, 3563 "variant": module.variantName, 3564 } 3565 err = headerTemplate.Execute(buf, infoMap) 3566 if err != nil { 3567 return err 3568 } 3569 3570 err = nw.Comment(buf.String()) 3571 if err != nil { 3572 return err 3573 } 3574 3575 err = nw.BlankLine() 3576 if err != nil { 3577 return err 3578 } 3579 3580 err = c.writeLocalBuildActions(nw, &module.actionDefs) 3581 if err != nil { 3582 return err 3583 } 3584 3585 err = nw.BlankLine() 3586 if err != nil { 3587 return err 3588 } 3589 } 3590 3591 return nil 3592} 3593 3594func (c *Context) writeAllSingletonActions(nw *ninjaWriter) error { 3595 headerTemplate := template.New("singletonHeader") 3596 _, err := headerTemplate.Parse(singletonHeaderTemplate) 3597 if err != nil { 3598 // This is a programming error. 3599 panic(err) 3600 } 3601 3602 buf := bytes.NewBuffer(nil) 3603 3604 for _, info := range c.singletonInfo { 3605 if len(info.actionDefs.variables)+len(info.actionDefs.rules)+len(info.actionDefs.buildDefs) == 0 { 3606 continue 3607 } 3608 3609 // Get the name of the factory function for the module. 3610 factory := info.factory 3611 factoryFunc := runtime.FuncForPC(reflect.ValueOf(factory).Pointer()) 3612 factoryName := factoryFunc.Name() 3613 3614 buf.Reset() 3615 infoMap := map[string]interface{}{ 3616 "name": info.name, 3617 "goFactory": factoryName, 3618 } 3619 err = headerTemplate.Execute(buf, infoMap) 3620 if err != nil { 3621 return err 3622 } 3623 3624 err = nw.Comment(buf.String()) 3625 if err != nil { 3626 return err 3627 } 3628 3629 err = nw.BlankLine() 3630 if err != nil { 3631 return err 3632 } 3633 3634 err = c.writeLocalBuildActions(nw, &info.actionDefs) 3635 if err != nil { 3636 return err 3637 } 3638 3639 err = nw.BlankLine() 3640 if err != nil { 3641 return err 3642 } 3643 } 3644 3645 return nil 3646} 3647 3648func (c *Context) writeLocalBuildActions(nw *ninjaWriter, 3649 defs *localBuildActions) error { 3650 3651 // Write the local variable assignments. 3652 for _, v := range defs.variables { 3653 // A localVariable doesn't need the package names or config to 3654 // determine its name or value. 3655 name := v.fullName(nil) 3656 value, err := v.value(nil) 3657 if err != nil { 3658 panic(err) 3659 } 3660 err = nw.Assign(name, value.Value(c.pkgNames)) 3661 if err != nil { 3662 return err 3663 } 3664 } 3665 3666 if len(defs.variables) > 0 { 3667 err := nw.BlankLine() 3668 if err != nil { 3669 return err 3670 } 3671 } 3672 3673 // Write the local rules. 3674 for _, r := range defs.rules { 3675 // A localRule doesn't need the package names or config to determine 3676 // its name or definition. 3677 name := r.fullName(nil) 3678 def, err := r.def(nil) 3679 if err != nil { 3680 panic(err) 3681 } 3682 3683 err = def.WriteTo(nw, name, c.pkgNames) 3684 if err != nil { 3685 return err 3686 } 3687 3688 err = nw.BlankLine() 3689 if err != nil { 3690 return err 3691 } 3692 } 3693 3694 // Write the build definitions. 3695 for _, buildDef := range defs.buildDefs { 3696 err := buildDef.WriteTo(nw, c.pkgNames) 3697 if err != nil { 3698 return err 3699 } 3700 3701 if len(buildDef.Args) > 0 { 3702 err = nw.BlankLine() 3703 if err != nil { 3704 return err 3705 } 3706 } 3707 } 3708 3709 return nil 3710} 3711 3712func beforeInModuleList(a, b *moduleInfo, list []*moduleInfo) bool { 3713 found := false 3714 if a == b { 3715 return false 3716 } 3717 for _, l := range list { 3718 if l == a { 3719 found = true 3720 } else if l == b { 3721 return found 3722 } 3723 } 3724 3725 missing := a 3726 if found { 3727 missing = b 3728 } 3729 panic(fmt.Errorf("element %v not found in list %v", missing, list)) 3730} 3731 3732type panicError struct { 3733 panic interface{} 3734 stack []byte 3735 in string 3736} 3737 3738func newPanicErrorf(panic interface{}, in string, a ...interface{}) error { 3739 buf := make([]byte, 4096) 3740 count := runtime.Stack(buf, false) 3741 return panicError{ 3742 panic: panic, 3743 in: fmt.Sprintf(in, a...), 3744 stack: buf[:count], 3745 } 3746} 3747 3748func (p panicError) Error() string { 3749 return fmt.Sprintf("panic in %s\n%s\n%s\n", p.in, p.panic, p.stack) 3750} 3751 3752func (p *panicError) addIn(in string) { 3753 p.in += " in " + in 3754} 3755 3756func funcName(f interface{}) string { 3757 return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() 3758} 3759 3760var fileHeaderTemplate = `****************************************************************************** 3761*** This file is generated and should not be edited *** 3762****************************************************************************** 3763{{if .Pkgs}} 3764This file contains variables, rules, and pools with name prefixes indicating 3765they were generated by the following Go packages: 3766{{range .Pkgs}} 3767 {{.PkgName}} [from Go package {{.PkgPath}}]{{end}}{{end}} 3768 3769` 3770 3771var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3772Module: {{.name}} 3773Variant: {{.variant}} 3774Type: {{.typeName}} 3775Factory: {{.goFactory}} 3776Defined: {{.pos}} 3777` 3778 3779var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3780Singleton: {{.name}} 3781Factory: {{.goFactory}} 3782` 3783