]>
git.ipfire.org Git - thirdparty/openembedded/openembedded-core.git/blob - scripts/combo-layer
2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright 2011 Intel Corporation
6 # Authored-by: Yu Ke <ke.yu@intel.com>
7 # Paul Eggleton <paul.eggleton@intel.com>
8 # Richard Purdie <richard.purdie@intel.com>
10 # SPDX-License-Identifier: GPL-2.0-only
24 from string
import Template
25 from functools
import reduce
30 logger
= logging
. getLogger ( "" )
31 loggerhandler
= logging
. StreamHandler ()
32 loggerhandler
. setFormatter ( logging
. Formatter ( "[ %(asctime)s ] %(message)s " , "%H:%M:%S" ))
33 logger
. addHandler ( loggerhandler
)
34 logger
. setLevel ( logging
. INFO
)
37 logger
= logger_create ()
39 def get_current_branch ( repodir
= None ):
41 if not os
. path
. exists ( os
. path
. join ( repodir
if repodir
else '' , ".git" )):
42 # Repo not created yet (i.e. during init) so just assume master
44 branchname
= runcmd ( "git symbolic-ref HEAD 2>/dev/null" , repodir
). strip ()
45 if branchname
. startswith ( "refs/heads/" ):
46 branchname
= branchname
[ 11 :]
48 except subprocess
. CalledProcessError
:
51 class Configuration ( object ):
53 Manages the configuration
55 For an example config file, see combo-layer.conf.example
58 def __init__ ( self
, options
):
59 for key
, val
in options
.__ dict
__ . items ():
60 setattr ( self
, key
, val
)
62 def readsection ( parser
, section
, repo
):
63 for ( name
, value
) in parser
. items ( section
):
64 if value
. startswith ( "@" ):
65 self
. repos
[ repo
][ name
] = eval ( value
. strip ( "@" ))
67 # Apply special type transformations for some properties.
68 # Type matches the RawConfigParser.get*() methods.
69 types
= { 'signoff' : 'boolean' , 'update' : 'boolean' , 'history' : 'boolean' }
71 value
= getattr ( parser
, 'get' + types
[ name
])( section
, name
)
72 self
. repos
[ repo
][ name
] = value
74 def readglobalsection ( parser
, section
):
75 for ( name
, value
) in parser
. items ( section
):
76 if name
== "commit_msg" :
77 self
. commit_msg_template
= value
79 logger
. debug ( "Loading config file %s " % self
. conffile
)
80 self
. parser
= configparser
. ConfigParser ()
81 with
open ( self
. conffile
) as f
:
82 self
. parser
. read_file ( f
)
84 # initialize default values
85 self
. commit_msg_template
= "Automatic commit to update last_revision"
88 for repo
in self
. parser
. sections ():
89 if repo
== "combo-layer-settings" :
90 # special handling for global settings
91 readglobalsection ( self
. parser
, repo
)
94 readsection ( self
. parser
, repo
, repo
)
96 # Load local configuration, if available
97 self
. localconffile
= None
98 self
. localparser
= None
99 self
. combobranch
= None
100 if self
. conffile
. endswith ( '.conf' ):
101 lcfile
= self
. conffile
. replace ( '.conf' , '-local.conf' )
102 if os
. path
. exists ( lcfile
):
103 # Read combo layer branch
104 self
. combobranch
= get_current_branch ()
105 logger
. debug ( "Combo layer branch is %s " % self
. combobranch
)
107 self
. localconffile
= lcfile
108 logger
. debug ( "Loading local config file %s " % self
. localconffile
)
109 self
. localparser
= configparser
. ConfigParser ()
110 with
open ( self
. localconffile
) as f
:
111 self
. localparser
. readfp ( f
)
113 for section
in self
. localparser
. sections ():
115 sectionvals
= section
. split ( '|' )
116 repo
= sectionvals
[ 0 ]
117 if sectionvals
[ 1 ] != self
. combobranch
:
121 if repo
in self
. repos
:
122 readsection ( self
. localparser
, section
, repo
)
124 def update ( self
, repo
, option
, value
, initmode
= False ):
125 # If the main config has the option already, that is what we
126 # are expected to modify.
127 if self
. localparser
and not self
. parser
. has_option ( repo
, option
):
128 parser
= self
. localparser
129 section
= " %s | %s " % ( repo
, self
. combobranch
)
130 conffile
= self
. localconffile
131 if initmode
and not parser
. has_section ( section
):
132 parser
. add_section ( section
)
136 conffile
= self
. conffile
137 parser
. set ( section
, option
, value
)
138 with
open ( conffile
, "w" ) as f
:
140 self
. repos
[ repo
][ option
] = value
142 def sanity_check ( self
, initmode
= False ):
143 required_options
=[ "src_uri" , "local_repo_dir" , "dest_dir" , "last_revision" ]
145 required_options
. remove ( "last_revision" )
148 for name
in self
. repos
:
149 for option
in required_options
:
150 if option
not in self
. repos
[ name
]:
151 msg
= " %s \n Option %s is not defined for component %s " %( msg
, option
, name
)
152 missing_options
. append ( option
)
153 # Sanitize dest_dir so that we do not have to deal with edge cases
154 # (unset, empty string, double slashes) in the rest of the code.
155 # It not being set will still be flagged as error because it is
156 # listed as required option above; that could be changed now.
157 dest_dir
= os
. path
. normpath ( self
. repos
[ name
]. get ( "dest_dir" , "." ))
158 self
. repos
[ name
][ "dest_dir" ] = "." if not dest_dir
else dest_dir
160 logger
. error ( "configuration file %s has the following error: %s " % ( self
. conffile
, msg
))
161 if self
. localconffile
and 'last_revision' in missing_options
:
162 logger
. error ( "local configuration file %s may be missing configuration for combo branch %s " % ( self
. localconffile
, self
. combobranch
))
165 # filterdiff is required by action_splitpatch, so check its availability
166 if subprocess
. call ( "which filterdiff > /dev/null 2>&1" , shell
= True ) != 0 :
167 logger
. error ( "ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)" )
170 def runcmd ( cmd
, destdir
= None , printerr
= True , out
= None , env
= None ):
172 execute command, raise CalledProcessError if fail
173 return output if succeed
175 logger
. debug ( "run cmd ' %s ' in %s " % ( cmd
, os
. getcwd () if destdir
is None else destdir
))
177 out
= tempfile
. TemporaryFile ()
180 err
= tempfile
. TemporaryFile ()
182 subprocess
. check_call ( cmd
, stdout
= out
, stderr
= err
, cwd
= destdir
, shell
= isinstance ( cmd
, str ), env
= env
or os
. environ
)
183 except subprocess
. CalledProcessError
as e
:
186 logger
. error ( " %s " % err
. read ())
190 output
= err
. read (). decode ( 'utf-8' )
191 logger
. debug ( "output: %s " % output
. replace ( chr ( 0 ), ' \\ 0' ))
194 def action_sync_revs ( conf
, args
):
196 Update the last_revision config option for each repo with the latest
197 revision in the remote's branch. Useful if multiple people are using
200 repos
= get_repos ( conf
, args
[ 1 :])
203 repo
= conf
. repos
[ name
]
204 ldir
= repo
[ 'local_repo_dir' ]
205 branch
= repo
. get ( 'branch' , "master" )
206 runcmd ( "git fetch" , ldir
)
207 lastrev
= runcmd ( 'git rev-parse origin/ %s ' % branch
, ldir
). strip ()
208 print ( "Updating %s to %s " % ( name
, lastrev
))
209 conf
. update ( name
, "last_revision" , lastrev
)
211 def action_init ( conf
, args
):
213 Clone component repositories
214 Check git is initialised; if not, copy initial data from component repos
216 for name
in conf
. repos
:
217 ldir
= conf
. repos
[ name
][ 'local_repo_dir' ]
218 if not os
. path
. exists ( ldir
):
219 logger
. info ( "cloning %s to %s " %( conf
. repos
[ name
][ 'src_uri' ], ldir
))
220 subprocess
. check_call ( "git clone %s %s " % ( conf
. repos
[ name
][ 'src_uri' ], ldir
), shell
= True )
221 if not os
. path
. exists ( ".git" ):
224 # Need a common ref for all trees.
225 runcmd ( 'git commit -m "initial empty commit" --allow-empty' )
226 startrev
= runcmd ( 'git rev-parse master' ). strip ()
228 for name
in conf
. repos
:
229 repo
= conf
. repos
[ name
]
230 ldir
= repo
[ 'local_repo_dir' ]
231 branch
= repo
. get ( 'branch' , "master" )
232 lastrev
= repo
. get ( 'last_revision' , None )
233 if lastrev
and lastrev
!= "HEAD" :
236 if not check_rev_branch ( name
, ldir
, lastrev
, branch
):
238 logger
. info ( "Copying data from %s at specified revision %s ..." % ( name
, lastrev
))
242 logger
. info ( "Copying data from %s ..." % name
)
243 # Sanity check initialrev and turn it into hash (required for copying history,
244 # because resolving a name ref only works in the component repo).
245 rev
= runcmd ( 'git rev-parse %s ' % initialrev
, ldir
). strip ()
246 if rev
!= initialrev
:
248 refs
= runcmd ( 'git show-ref -s %s ' % initialrev
, ldir
). split ( ' \n ' )
249 if len ( set ( refs
)) > 1 :
250 # Happens for example when configured to track
251 # "master" and there is a refs/heads/master. The
252 # traditional behavior from "git archive" (preserved
253 # here) it to choose the first one. This might not be
254 # intended, so at least warn about it.
255 logger
. warning ( " %s : initial revision ' %s ' not unique, picking result of rev-parse = %s " %
256 ( name
, initialrev
, refs
[ 0 ]))
259 # show-ref fails for hashes. Skip the sanity warning in that case.
262 dest_dir
= repo
[ 'dest_dir' ]
264 extract_dir
= os
. path
. join ( os
. getcwd (), dest_dir
)
265 if not os
. path
. exists ( extract_dir
):
266 os
. makedirs ( extract_dir
)
268 extract_dir
= os
. getcwd ()
269 file_filter
= repo
. get ( 'file_filter' , "" )
270 exclude_patterns
= repo
. get ( 'file_exclude' , '' ). split ()
271 def copy_selected_files ( initialrev
, extract_dir
, file_filter
, exclude_patterns
, ldir
,
273 # When working inside a filtered branch which had the
274 # files already moved, we need to prepend the
275 # subdirectory to all filters, otherwise they would
280 subdir
= os
. path
. normpath ( subdir
)
281 file_filter
= ' ' . join ([ subdir
+ '/' + x
for x
in file_filter
. split ()])
282 exclude_patterns
= [ subdir
+ '/' + x
for x
in exclude_patterns
]
283 # To handle both cases, we cd into the target
284 # directory and optionally tell tar to strip the path
285 # prefix when the files were already moved.
286 subdir_components
= len ( subdir
. split ( os
. path
. sep
)) if subdir
else 0
287 strip
=( '--strip-components= %d ' % subdir_components
) if subdir
else ''
288 # TODO: file_filter wild cards do not work (and haven't worked before either), because
289 # a) GNU tar requires a --wildcards parameter before turning on wild card matching.
290 # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
291 # in contrast to the other use of file_filter as parameter of "git archive"
292 # where it only matches .c files directly in src).
293 files
= runcmd ( "git archive %s %s | tar -x -v %s -C %s %s " %
295 strip
, extract_dir
, file_filter
),
298 # Implement file removal by letting tar create the
299 # file and then deleting it in the file system
300 # again. Uses the list of files created by tar (easier
301 # than walking the tree).
302 for file in files
. split ( ' \n ' ):
303 if file . endswith ( os
. path
. sep
):
305 for pattern
in exclude_patterns
:
306 if fnmatch
. fnmatch ( file , pattern
):
307 os
. unlink ( os
. path
. join (*([ extract_dir
] + [ '..' ] * subdir_components
+ [ file ])))
311 copy_selected_files ( initialrev
, extract_dir
, file_filter
, exclude_patterns
, ldir
)
313 # First fetch remote history into local repository.
314 # We need a ref for that, so ensure that there is one.
315 refname
= "combo-layer-init- %s " % name
316 runcmd ( "git branch -f %s %s " % ( refname
, initialrev
), ldir
)
317 runcmd ( "git fetch %s %s " % ( ldir
, refname
))
318 runcmd ( "git branch -D %s " % refname
, ldir
)
319 # Make that the head revision.
320 runcmd ( "git checkout -b %s %s " % ( name
, initialrev
))
321 # Optional: cut the history by replacing the given
322 # start point(s) with commits providing the same
323 # content (aka tree), but with commit information that
324 # makes it clear that this is an artifically created
325 # commit and nothing the original authors had anything
327 since_rev
= repo
. get ( 'since_revision' , '' )
329 committer
= runcmd ( 'git var GIT_AUTHOR_IDENT' ). strip ()
330 # Same time stamp, no name.
331 author
= re
. sub ( '.* (\d+ [+-]\d+)' , r
'unknown <unknown> \1' , committer
)
332 logger
. info ( 'author %s ' % author
)
333 for rev
in since_rev
. split ():
334 # Resolve in component repo...
335 rev
= runcmd ( 'git log --oneline --no-abbrev-commit -n1 %s ' % rev
, ldir
). split ()[ 0 ]
336 # ... and then get the tree in current
337 # one. The commit should be in both repos with
338 # the same tree, but better check here.
339 tree
= runcmd ( 'git show -s --pretty=format: %% T %s ' % rev
). strip ()
340 with tempfile
. NamedTemporaryFile ( mode
= 'wt' ) as editor
:
341 editor
. write ( '''cat >$1 <<EOF
346 %s : squashed import of component
348 This commit copies the entire set of files as found in
351 For more information about previous commits, see the
354 Commit created by combo-layer.
356 ''' % ( tree
, author
, committer
, name
, name
, since_rev
))
358 os
. environ
[ 'GIT_EDITOR' ] = 'sh %s ' % editor
. name
359 runcmd ( 'git replace --edit %s ' % rev
)
361 # Optional: rewrite history to change commit messages or to move files.
362 if 'hook' in repo
or dest_dir
!= "." :
363 filter_branch
= [ 'git' , 'filter-branch' , '--force' ]
364 with tempfile
. NamedTemporaryFile ( mode
= 'wt' ) as hookwrapper
:
366 # Create a shell script wrapper around the original hook that
367 # can be used by git filter-branch. Hook may or may not have
370 hook
= os
. path
. join ( os
. path
. dirname ( conf
. conffile
), '..' , hook
)
371 # The wrappers turns the commit message
372 # from stdin into a fake patch header.
373 # This is good enough for changing Subject
374 # and commit msg body with normal
376 hookwrapper
. write ( '''set -e
378 trap "rm $tmpname" EXIT
379 echo -n 'Subject: [PATCH] ' >>$tmpname
381 if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
384 echo '---' >>$tmpname
385 %s $tmpname $GIT_COMMIT %s
386 tail -c +18 $tmpname | head -c -4
389 filter_branch
. extend ([ '--msg-filter' , 'bash %s ' % hookwrapper
. name
])
391 parent
= os
. path
. dirname ( dest_dir
)
394 # May run outside of the current directory, so do not assume that .git exists.
395 filter_branch
. extend ([ '--tree-filter' , 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s ' % ( parent
, dest_dir
)])
396 filter_branch
. append ( 'HEAD' )
397 runcmd ( filter_branch
)
398 runcmd ( 'git update-ref -d refs/original/refs/heads/ %s ' % name
)
399 repo
[ 'rewritten_revision' ] = runcmd ( 'git rev-parse HEAD' ). strip ()
400 repo
[ 'stripped_revision' ] = repo
[ 'rewritten_revision' ]
401 # Optional filter files: remove everything and re-populate using the normal filtering code.
402 # Override any potential .gitignore.
403 if file_filter
or exclude_patterns
:
404 runcmd ( 'git rm -rf .' )
405 if not os
. path
. exists ( extract_dir
):
406 os
. makedirs ( extract_dir
)
407 copy_selected_files ( 'HEAD' , extract_dir
, file_filter
, exclude_patterns
, '.' ,
409 runcmd ( 'git add --all --force .' )
410 if runcmd ( 'git status --porcelain' ):
411 # Something to commit.
412 runcmd ([ 'git' , 'commit' , '-m' ,
413 ''' %s : select file subset
415 Files from the component repository were chosen based on
416 the following filters:
418 file_exclude = %s ''' % ( name
, file_filter
or '<empty>' , repo
. get ( 'file_exclude' , '<empty>' ))])
419 repo
[ 'stripped_revision' ] = runcmd ( 'git rev-parse HEAD' ). strip ()
422 lastrev
= runcmd ( 'git rev-parse %s ' % initialrev
, ldir
). strip ()
423 conf
. update ( name
, "last_revision" , lastrev
, initmode
= True )
428 # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
429 runcmd ( 'git checkout master' )
430 merge
= [ 'git' , 'merge' , '--no-commit' ]
431 for name
in conf
. repos
:
432 repo
= conf
. repos
[ name
]
433 # Use branch created earlier.
435 # Root all commits which have no parent in the common
436 # ancestor in the new repository.
437 for start
in runcmd ( 'git log --pretty=format: %% H --max-parents=0 %s --' % name
). split ( ' \n ' ):
438 runcmd ( 'git replace --graft %s %s ' % ( start
, startrev
))
441 except Exception as error
:
442 logger
. info ( '''Merging component repository history failed, perhaps because of merge conflicts.
443 It may be possible to commit anyway after resolving these conflicts.
446 # Create MERGE_HEAD and MERGE_MSG. "git merge" itself
447 # does not create MERGE_HEAD in case of a (harmless) failure,
448 # and we want certain auto-generated information in the
449 # commit message for future reference and/or automation.
450 with
open ( '.git/MERGE_HEAD' , 'w' ) as head
:
451 with
open ( '.git/MERGE_MSG' , 'w' ) as msg
:
452 msg
. write ( 'repo: initial import of components \n\n ' )
453 # head.write('%s\n' % startrev)
454 for name
in conf
. repos
:
455 repo
= conf
. repos
[ name
]
456 # <upstream ref> <rewritten ref> <rewritten + files removed>
457 msg
. write ( 'combo-layer- %s : %s %s %s \n ' % ( name
,
458 repo
[ 'last_revision' ],
459 repo
[ 'rewritten_revision' ],
460 repo
[ 'stripped_revision' ]))
461 rev
= runcmd ( 'git rev-parse %s ' % name
). strip ()
462 head
. write ( ' %s \n ' % rev
)
464 if conf
. localconffile
:
467 runcmd ( "git rm --cached %s " % conf
. localconffile
, printerr
= False )
468 except subprocess
. CalledProcessError
:
471 localrelpath
= os
. path
. relpath ( conf
. localconffile
)
472 runcmd ( "grep -q %s .gitignore || echo %s >> .gitignore" % ( localrelpath
, localrelpath
))
473 runcmd ( "git add .gitignore" )
474 logger
. info ( "Added local configuration file %s to .gitignore" , localrelpath
)
475 logger
. info ( "Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit." )
477 logger
. info ( "Repository already initialised, nothing to do." )
480 def check_repo_clean ( repodir
):
482 check if the repo is clean
483 exit if repo is dirty
485 output
= runcmd ( "git status --porcelain" , repodir
)
486 r
= re
. compile ( r
'\?\? patch-.*/' )
487 dirtyout
= [ item
for item
in output
. splitlines () if not r
. match ( item
)]
489 logger
. error ( "git repo %s is dirty, please fix it first" , repodir
)
492 def check_patch ( patchfile
):
493 f
= open ( patchfile
, 'rb' )
506 elif ln
. startswith ( b
'--- ' ):
507 # We have a diff in the commit message
510 print ( 'WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile
)
511 of
= open ( patchfile
+ '.tmp' , 'wb' )
514 elif in_patch
and not ln
[ 0 ] in b
'+-@ \n\r ' :
527 os
. rename ( of
. name
, patchfile
)
529 def drop_to_shell ( workdir
= None ):
530 if not sys
. stdin
. isatty ():
531 print ( "Not a TTY so can't drop to shell for resolution, exiting." )
534 shell
= os
. environ
. get ( 'SHELL' , 'bash' )
535 print ( 'Dropping to shell " %s " \n ' \
536 'When you are finished, run the following to continue: \n ' \
537 ' exit -- continue to apply the patches \n ' \
538 ' exit 1 -- abort \n ' % shell
);
539 ret
= subprocess
. call ([ shell
], cwd
= workdir
)
546 def check_rev_branch ( component
, repodir
, rev
, branch
):
548 actualbranch
= runcmd ( "git branch --contains %s " % rev
, repodir
, printerr
= False )
549 except subprocess
. CalledProcessError
as e
:
550 if e
. returncode
== 129 :
556 logger
. error ( " %s : specified revision %s is invalid!" % ( component
, rev
))
560 branchlist
= actualbranch
. split ( " \n " )
562 branches
. append ( b
. strip (). split ( ' ' )[- 1 ])
564 if branch
not in branches
:
565 logger
. error ( " %s : specified revision %s is not on specified branch %s !" % ( component
, rev
, branch
))
569 def get_repos ( conf
, repo_names
):
571 for name
in repo_names
:
572 if name
. startswith ( '-' ):
577 if not repo
in conf
. repos
:
578 logger
. error ( "Specified component ' %s ' not found in configuration" % repo
)
582 repos
= [ repo
for repo
in conf
. repos
if conf
. repos
[ repo
]. get ( "update" , True ) ]
586 def action_pull ( conf
, args
):
588 update the component repos only
590 repos
= get_repos ( conf
, args
[ 1 :])
592 # make sure all repos are clean
594 check_repo_clean ( conf
. repos
[ name
][ 'local_repo_dir' ])
597 repo
= conf
. repos
[ name
]
598 ldir
= repo
[ 'local_repo_dir' ]
599 branch
= repo
. get ( 'branch' , "master" )
600 logger
. info ( "update branch %s of component repo %s in %s ..." % ( branch
, name
, ldir
))
601 if not conf
. hard_reset
:
602 # Try to pull only the configured branch. Beware that this may fail
603 # when the branch is currently unknown (for example, after reconfiguring
604 # combo-layer). In that case we need to fetch everything and try the check out
607 runcmd ( "git checkout %s " % branch
, ldir
, printerr
= False )
608 except subprocess
. CalledProcessError
:
609 output
= runcmd ( "git fetch" , ldir
)
611 runcmd ( "git checkout %s " % branch
, ldir
)
612 runcmd ( "git pull --ff-only" , ldir
)
614 output
= runcmd ( "git pull --ff-only" , ldir
)
617 output
= runcmd ( "git fetch" , ldir
)
619 runcmd ( "git checkout %s " % branch
, ldir
)
620 runcmd ( "git reset --hard FETCH_HEAD" , ldir
)
622 def action_update ( conf
, args
):
624 update the component repos
626 generate the patch list
627 apply the generated patches
629 re-creates the entire component history and merges them
630 into the current branch with a merge commit
632 components
= [ arg
. split ( ':' )[ 0 ] for arg
in args
[ 1 :]]
636 a
= arg
. split ( ':' , 1 )
637 revisions
[ a
[ 0 ]] = a
[ 1 ]
638 repos
= get_repos ( conf
, components
)
640 # make sure combo repo is clean
641 check_repo_clean ( os
. getcwd ())
643 # Check whether we keep the component histories. Must be
644 # set either via --history command line parameter or consistently
645 # in combo-layer.conf. Mixing modes is (currently, and probably
646 # permanently because it would be complicated) not supported.
652 repo
= conf
. repos
[ name
]
653 repo_history
= repo
. get ( 'history' , False )
655 history
= repo_history
656 elif history
!= repo_history
:
657 logger
. error ( "'history' property is set inconsistently" )
660 # Step 1: update the component repos
662 logger
. info ( "Skipping pull (-n)" )
664 action_pull ( conf
, [ 'arg0' ] + components
)
667 update_with_history ( conf
, components
, revisions
, repos
)
669 update_with_patches ( conf
, components
, revisions
, repos
)
671 def update_with_patches ( conf
, components
, revisions
, repos
):
673 patch_dir
= "patch- %s " % uuid
. uuid4 ()
674 if not os
. path
. exists ( patch_dir
):
678 revision
= revisions
. get ( name
, None )
679 repo
= conf
. repos
[ name
]
680 ldir
= repo
[ 'local_repo_dir' ]
681 dest_dir
= repo
[ 'dest_dir' ]
682 branch
= repo
. get ( 'branch' , "master" )
683 repo_patch_dir
= os
. path
. join ( os
. getcwd (), patch_dir
, name
)
685 # Step 2: generate the patch list and store to patch dir
686 logger
. info ( "Generating patches from %s ..." % name
)
687 top_revision
= revision
or branch
688 if not check_rev_branch ( name
, ldir
, top_revision
, branch
):
691 prefix
= "--src-prefix=a/ %s / --dst-prefix=b/ %s /" % ( dest_dir
, dest_dir
)
694 if repo
[ 'last_revision' ] == "" :
695 logger
. info ( "Warning: last_revision of component %s is not set, starting from the first commit" % name
)
696 patch_cmd_range
= "--root %s " % top_revision
697 rev_cmd_range
= top_revision
699 if not check_rev_branch ( name
, ldir
, repo
[ 'last_revision' ], branch
):
701 patch_cmd_range
= " %s .. %s " % ( repo
[ 'last_revision' ], top_revision
)
702 rev_cmd_range
= patch_cmd_range
704 file_filter
= repo
. get ( 'file_filter' , "." )
706 # Filter out unwanted files
707 exclude
= repo
. get ( 'file_exclude' , '' )
709 for path
in exclude
. split ():
710 p
= " %s / %s " % ( dest_dir
, path
) if dest_dir
!= '.' else path
711 file_filter
+= " ':! %s '" % p
713 patch_cmd
= "git format-patch -N %s --output-directory %s %s -- %s " % \
714 ( prefix
, repo_patch_dir
, patch_cmd_range
, file_filter
)
715 output
= runcmd ( patch_cmd
, ldir
)
716 logger
. debug ( "generated patch set: \n %s " % output
)
717 patchlist
= output
. splitlines ()
719 rev_cmd
= "git rev-list --no-merges %s -- %s " % ( rev_cmd_range
, file_filter
)
720 revlist
= runcmd ( rev_cmd
, ldir
). splitlines ()
722 # Step 3: Call repo specific hook to adjust patch
724 # hook parameter is: ./hook patchpath revision reponame
726 for patch
in patchlist
:
727 runcmd ( " %s %s %s %s " % ( repo
[ 'hook' ], patch
, revlist
[ count
], name
))
730 # Step 4: write patch list and revision list to file, for user to edit later
731 patchlist_file
= os
. path
. join ( os
. getcwd (), patch_dir
, "patchlist- %s " % name
)
732 repo
[ 'patchlist' ] = patchlist_file
733 f
= open ( patchlist_file
, 'w' )
735 for patch
in patchlist
:
736 f
. write ( " %s %s \n " % ( patch
, revlist
[ count
]))
737 check_patch ( os
. path
. join ( patch_dir
, patch
))
741 # Step 5: invoke bash for user to edit patch and patch list
743 print ( 'You may now edit the patch and patch list in %s \n ' \
744 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir
);
745 if not drop_to_shell ( patch_dir
):
748 # Step 6: apply the generated and revised patch
749 apply_patchlist ( conf
, repos
)
750 runcmd ( "rm -rf %s " % patch_dir
)
752 # Step 7: commit the updated config file if it's being tracked
753 commit_conf_file ( conf
, components
)
755 def conf_commit_msg ( conf
, components
):
756 # create the "components" string
757 component_str
= "all components"
758 if len ( components
) > 0 :
759 # otherwise tell which components were actually changed
760 component_str
= ", " . join ( components
)
762 # expand the template with known values
763 template
= Template ( conf
. commit_msg_template
)
764 msg
= template
. substitute ( components
= component_str
)
767 def commit_conf_file ( conf
, components
, commit
= True ):
768 relpath
= os
. path
. relpath ( conf
. conffile
)
770 output
= runcmd ( "git status --porcelain %s " % relpath
, printerr
= False )
772 # Outside the repository
775 if output
. lstrip (). startswith ( "M" ):
776 logger
. info ( "Committing updated configuration file" )
778 msg
= conf_commit_msg ( conf
, components
)
779 runcmd ( 'git commit -m' . split () + [ msg
, relpath
])
781 runcmd ( 'git add %s ' % relpath
)
785 def apply_patchlist ( conf
, repos
):
787 apply the generated patch list to combo repo
790 repo
= conf
. repos
[ name
]
791 lastrev
= repo
[ "last_revision" ]
794 # Get non-blank lines from patch list file
796 if os
. path
. exists ( repo
[ 'patchlist' ]) or not conf
. interactive
:
797 # Note: we want this to fail here if the file doesn't exist and we're not in
798 # interactive mode since the file should exist in this case
799 with
open ( repo
[ 'patchlist' ]) as f
:
803 patchlist
. append ( line
)
805 ldir
= conf
. repos
[ name
][ 'local_repo_dir' ]
806 branch
= conf
. repos
[ name
]. get ( 'branch' , "master" )
807 branchrev
= runcmd ( "git rev-parse %s " % branch
, ldir
). strip ()
810 logger
. info ( "Applying patches from %s ..." % name
)
811 linecount
= len ( patchlist
)
813 for line
in patchlist
:
814 patchfile
= line
. split ()[ 0 ]
815 lastrev
= line
. split ()[ 1 ]
816 patchdisp
= os
. path
. relpath ( patchfile
)
817 if os
. path
. getsize ( patchfile
) == 0 :
818 logger
. info ( "(skipping %d / %d %s - no changes)" % ( i
, linecount
, patchdisp
))
820 cmd
= "git am --keep-cr %s- p1 %s " % ( '-s ' if repo
. get ( 'signoff' , True ) else '' , patchfile
)
821 logger
. info ( "Applying %d / %d : %s " % ( i
, linecount
, patchdisp
))
824 except subprocess
. CalledProcessError
:
825 logger
. info ( 'Running "git am --abort" to cleanup repo' )
826 runcmd ( "git am --abort" )
827 logger
. error ( '" %s " failed' % cmd
)
828 logger
. info ( "Please manually apply patch %s " % patchdisp
)
829 logger
. info ( "Note: if you exit and continue applying without manually applying the patch, it will be skipped" )
830 if not drop_to_shell ():
831 if prevrev
!= repo
[ 'last_revision' ]:
832 conf
. update ( name
, "last_revision" , prevrev
)
836 # Once all patches are applied, we should update
837 # last_revision to the branch head instead of the last
838 # applied patch. The two are not necessarily the same when
839 # the last commit is a merge commit or when the patches at
840 # the branch head were intentionally excluded.
842 # If we do not do that for a merge commit, the next
843 # combo-layer run will only exclude patches reachable from
844 # one of the merged branches and try to re-apply patches
845 # from other branches even though they were already
848 # If patches were intentionally excluded, the next run will
849 # present them again instead of skipping over them. This
850 # may or may not be intended, so the code here is conservative
851 # and only addresses the "head is merge commit" case.
852 if lastrev
!= branchrev
and \
853 len ( runcmd ( "git show --pretty=format: %% P --no-patch %s " % branch
, ldir
). split ()) > 1 :
856 logger
. info ( "No patches to apply from %s " % name
)
859 if lastrev
!= repo
[ 'last_revision' ]:
860 conf
. update ( name
, "last_revision" , lastrev
)
862 def action_splitpatch ( conf
, args
):
864 generate the commit patch and
865 split the patch per repo
867 logger
. debug ( "action_splitpatch" )
872 patchdir
= "splitpatch- %s " % commit
873 if not os
. path
. exists ( patchdir
):
876 # filerange_root is for the repo whose dest_dir is root "."
877 # and it should be specified by excluding all other repo dest dir
878 # like "-x repo1 -x repo2 -x repo3 ..."
880 for name
in conf
. repos
:
881 dest_dir
= conf
. repos
[ name
][ 'dest_dir' ]
883 filerange_root
= ' %s -x " %s /*"' % ( filerange_root
, dest_dir
)
885 for name
in conf
. repos
:
886 dest_dir
= conf
. repos
[ name
][ 'dest_dir' ]
887 patch_filename
= " %s / %s .patch" % ( patchdir
, name
)
889 cmd
= "git format-patch -n1 --stdout %s ^.. %s | filterdiff -p1 %s > %s " % ( commit
, commit
, filerange_root
, patch_filename
)
891 cmd
= "git format-patch --no-prefix -n1 --stdout %s ^.. %s -- %s > %s " % ( commit
, commit
, dest_dir
, patch_filename
)
893 # Detect empty patches (including those produced by filterdiff above
894 # that contain only preamble text)
895 if os
. path
. getsize ( patch_filename
) == 0 or runcmd ( "filterdiff %s " % patch_filename
) == "" :
896 os
. remove ( patch_filename
)
897 logger
. info ( "(skipping %s - no changes)" , name
)
899 logger
. info ( patch_filename
)
901 def update_with_history ( conf
, components
, revisions
, repos
):
902 '''Update all components with full history.
904 Works by importing all commits reachable from a component's
905 current head revision. If those commits are rooted in an already
906 imported commit, their content gets mixed with the content of the
907 combined repo of that commit (new or modified files overwritten,
908 removed files removed).
910 The last commit is an artificial merge commit that merges all the
911 updated components into the combined repository.
913 The HEAD ref only gets updated at the very end. All intermediate work
914 happens in a worktree which will get garbage collected by git eventually
917 # Remember current HEAD and what we need to add to it.
918 head
= runcmd ( "git rev-parse HEAD" ). strip ()
919 additional_heads
= {}
921 # Track the mapping between original commit and commit in the
922 # combined repo. We do not have to distinguish between components,
923 # because commit hashes are different anyway. Often we can
924 # skip find_revs() entirely (for example, when all new commits
925 # are derived from the last imported revision).
927 # Using "head" (typically the merge commit) instead of the actual
928 # commit for the component leads to a nicer history in the combined
932 repo
= conf
. repos
[ name
]
933 revision
= repo
[ 'last_revision' ]
935 old2new_revs
[ revision
] = head
938 '''Insert -p before each entry.'''
941 parameters
. append ( '-p' )
945 # Do all intermediate work with a separate work dir and index,
946 # chosen via env variables (can't use "git worktree", it is too
947 # new). This is useful (no changes to current work tree unless the
948 # update succeeds) and required (otherwise we end up temporarily
949 # removing the combo-layer hooks that we currently use when
950 # importing a new component).
952 # Not cleaned up after a failure at the moment.
953 wdir
= os
. path
. join ( os
. getcwd (), ".git" , "combo-layer" )
954 windex
= wdir
+ ".index"
955 if os
. path
. isdir ( wdir
):
958 wenv
= copy
. deepcopy ( os
. environ
)
959 wenv
[ "GIT_WORK_TREE" ] = wdir
960 wenv
[ "GIT_INDEX_FILE" ] = windex
961 # This one turned out to be needed in practice.
962 wenv
[ "GIT_OBJECT_DIRECTORY" ] = os
. path
. join ( os
. getcwd (), ".git" , "objects" )
963 wargs
= { "destdir" : wdir
, "env" : wenv
}
966 revision
= revisions
. get ( name
, None )
967 repo
= conf
. repos
[ name
]
968 ldir
= repo
[ 'local_repo_dir' ]
969 dest_dir
= repo
[ 'dest_dir' ]
970 branch
= repo
. get ( 'branch' , "master" )
971 hook
= repo
. get ( 'hook' , None )
972 largs
= { "destdir" : ldir
, "env" : None }
973 file_include
= repo
. get ( 'file_filter' , '' ). split ()
974 file_include
. sort () # make sure that short entries like '.' come first.
975 file_exclude
= repo
. get ( 'file_exclude' , '' ). split ()
977 def include_file ( file ):
979 # No explicit filter set, include file.
981 for filter in file_include
:
983 # Another special case: include current directory and thus all files.
985 if os
. path
. commonprefix (( filter , file )) == filter :
986 # Included in directory or direct file match.
988 # Check for wildcard match *with* allowing * to match /, i.e.
989 # src/*.c does match src/foobar/*.c. That's not how it is done elsewhere
990 # when passing the filtering to "git archive", but it is unclear what
991 # the intended semantic is (the comment on file_exclude that "append a * wildcard
992 # at the end" to match the full content of a directories implies that
993 # slashes are indeed not special), so here we simply do what's easy to
994 # implement in Python.
995 logger
. debug ( 'fnmatch( %s , %s )' % ( file , filter ))
996 if fnmatch
. fnmatchcase ( file , filter ):
1000 def exclude_file ( file ):
1001 for filter in file_exclude
:
1002 if fnmatch
. fnmatchcase ( file , filter ):
1006 def file_filter ( files
):
1007 '''Clean up file list so that only included files remain.'''
1009 while index
< len ( files
):
1011 if not include_file ( file ) or exclude_file ( file ):
1017 # Generate the revision list.
1018 logger
. info ( "Analyzing commits from %s ..." % name
)
1019 top_revision
= revision
or branch
1020 if not check_rev_branch ( name
, ldir
, top_revision
, branch
):
1023 last_revision
= repo
[ 'last_revision' ]
1024 rev_list_args
= "--full-history --sparse --topo-order --reverse"
1025 if not last_revision
:
1026 logger
. info ( "Warning: last_revision of component %s is not set, starting from the first commit" % name
)
1027 rev_list_args
= rev_list_args
+ ' ' + top_revision
1029 if not check_rev_branch ( name
, ldir
, last_revision
, branch
):
1031 rev_list_args
= " %s %s .. %s " % ( rev_list_args
, last_revision
, top_revision
)
1033 # By definition, the current HEAD contains the latest imported
1034 # commit of each component. We use that as initial mapping even
1035 # though the commits do not match exactly because
1036 # a) it always works (in contrast to find_revs, which relies on special
1038 # b) it is faster than find_revs, which will only be called on demand
1039 # and can be skipped entirely in most cases
1040 # c) last but not least, the combined history looks nicer when all
1041 # new commits are rooted in the same merge commit
1042 old2new_revs
[ last_revision
] = head
1044 # We care about all commits (--full-history and --sparse) and
1045 # we want reconstruct the topology and thus do not care
1046 # about ordering by time (--topo-order). We ask for the ones
1047 # we need to import first to be listed first (--reverse).
1048 revs
= runcmd ( "git rev-list %s " % rev_list_args
, ** largs
). split ()
1049 logger
. debug ( "To be imported: %s " % revs
)
1050 # Now 'revs' contains all revisions reachable from the top revision.
1051 # All revisions derived from the 'last_revision' definitely are new,
1052 # whereas the others may or may not have been imported before. For
1053 # a linear history in the component, that second set will be empty.
1054 # To distinguish between them, we also get the shorter list
1055 # of revisions starting at the ancestor.
1057 ancestor_revs
= runcmd ( "git rev-list --ancestry-path %s " % rev_list_args
, ** largs
). split ()
1060 logger
. debug ( "Ancestors: %s " % ancestor_revs
)
1062 # Now import each revision.
1063 logger
. info ( "Importing commits from %s ..." % name
)
1064 def import_rev ( rev
):
1067 # If it is part of the new commits, we definitely need
1068 # to import it. Otherwise we need to check, we might have
1069 # imported it before. If it was imported and we merely
1070 # fail to find it because commit messages did not track
1071 # the mapping, then we end up importing it again. So
1072 # combined repos using "updating with history" really should
1073 # enable the "From ... rev:" commit header modifications.
1074 if rev
not in ancestor_revs
and rev
not in old2new_revs
and not scanned_revs
:
1075 logger
. debug ( "Revision %s triggers log analysis." % rev
)
1076 find_revs ( old2new_revs
, head
)
1078 new_rev
= old2new_revs
. get ( rev
, None )
1082 # If the commit is not in the original list of revisions
1083 # to be imported, then it must be a parent of one of those
1084 # commits and it was skipped during earlier imports or not
1085 # found. Importing such merge commits leads to very ugly
1086 # history (long cascade of merge commits which all point
1087 # to to older commits) when switching from "update via
1088 # patches" to "update with history".
1090 # We can avoid importing merge commits if all non-merge commits
1091 # reachable from it were already imported. In that case we
1092 # can root the new commits in the current head revision.
1093 def is_imported ( prev
):
1094 parents
= runcmd ( "git show --no-patch --pretty=format:%P " + prev
, ** largs
). split ()
1095 if len ( parents
) > 1 :
1097 if not is_imported ( p
):
1098 logger
. debug ( "Must import %s because %s is not imported." % ( rev
, p
))
1101 elif prev
in old2new_revs
:
1104 logger
. debug ( "Must import %s because %s is not imported." % ( rev
, prev
))
1106 if rev
not in revs
and is_imported ( rev
):
1107 old2new_revs
[ rev
] = head
1110 # Need to import rev. Collect some information about it.
1111 logger
. debug ( "Importing %s " % rev
)
1112 ( parents
, author_name
, author_email
, author_timestamp
, body
) = \
1113 runcmd ( "git show --no-patch --pretty=format:%P %x00% an %x00% ae %x00% at %x00% B " + rev
, ** largs
). split ( chr ( 0 ))
1114 parents
= parents
. split ()
1116 # Arbitrarily pick the first parent as base. It may or may not have
1117 # been imported before. For example, if the parent is a merge commit
1118 # and previously the combined repository used patching as update
1119 # method, then the actual merge commit parent never was imported.
1120 # To cover this, We recursively import parents.
1122 new_parent
= import_rev ( parent
)
1123 # Clean index and working tree. TODO: can we combine this and the
1124 # next into one command with less file IO?
1125 # "git reset --hard" does not work, it changes HEAD of the parent
1126 # repo, which we wanted to avoid. Probably need to keep
1127 # track of the rev that corresponds to the index and use apply_commit().
1128 runcmd ( "git rm -q --ignore-unmatch -rf ." , ** wargs
)
1129 # Update index and working tree to match the parent.
1130 runcmd ( "git checkout -q -f %s ." % new_parent
, ** wargs
)
1133 # Clean index and working tree.
1134 runcmd ( "git rm -q --ignore-unmatch -rf ." , ** wargs
)
1136 # Modify index and working tree such that it mirrors the commit.
1137 apply_commit ( parent
, rev
, largs
, wargs
, dest_dir
, file_filter
= file_filter
)
1140 new_tree
= runcmd ( "git write-tree" , ** wargs
). strip ()
1141 env
= copy
. deepcopy ( wenv
)
1142 env
[ 'GIT_AUTHOR_NAME' ] = author_name
1143 env
[ 'GIT_AUTHOR_EMAIL' ] = author_email
1144 env
[ 'GIT_AUTHOR_DATE' ] = author_timestamp
1146 # Need to turn the verbatim commit message into something resembling a patch header
1148 with tempfile
. NamedTemporaryFile ( mode
= 'wt' , delete
= False ) as patch
:
1149 patch
. write ( 'Subject: [PATCH] ' )
1151 patch
. write ( ' \n --- \n ' )
1153 runcmd ([ hook
, patch
. name
, rev
, name
])
1154 with
open ( patch
. name
) as f
:
1155 body
= f
. read ()[ len ( 'Subject: [PATCH] ' ):][:- len ( ' \n --- \n ' )]
1157 # We can skip non-merge commits that did not change any files. Those are typically
1158 # the result of file filtering, although they could also have been introduced
1159 # intentionally upstream, in which case we drop some information here.
1160 if len ( parents
) == 1 :
1161 parent_rev
= import_rev ( parents
[ 0 ])
1162 old_tree
= runcmd ( "git show -s --pretty=format:%T " + parent_rev
, ** wargs
). strip ()
1163 commit
= old_tree
!= new_tree
1165 new_rev
= parent_rev
1169 new_rev
= runcmd ( "git commit-tree" . split () + add_p ([ import_rev ( p
) for p
in parents
]) +
1170 [ "-m" , body
, new_tree
],
1172 old2new_revs
[ rev
] = new_rev
1179 # Remember how to update our current head. New components get added,
1180 # updated components get the delta between current head and the updated component
1182 additional_heads
[ old2new_revs
[ revs
[- 1 ]]] = head
if repo
[ 'last_revision' ] else None
1183 repo
[ 'last_revision' ] = revs
[- 1 ]
1185 # Now construct the final merge commit. We create the tree by
1186 # starting with the head and applying the changes from each
1187 # components imported head revision.
1188 if additional_heads
:
1189 runcmd ( "git reset --hard" , ** wargs
)
1190 for rev
, base
in additional_heads
. items ():
1191 apply_commit ( base
, rev
, wargs
, wargs
, None )
1193 # Commit with all component branches as parents as well as the previous head.
1194 logger
. info ( "Writing final merge commit..." )
1195 msg
= conf_commit_msg ( conf
, components
)
1196 new_tree
= runcmd ( "git write-tree" , ** wargs
). strip ()
1197 new_rev
= runcmd ( "git commit-tree" . split () +
1198 add_p ([ head
] + list ( additional_heads
. keys ())) +
1199 [ "-m" , msg
, new_tree
],
1201 # And done! This is the first time we change the HEAD in the actual work tree.
1202 runcmd ( "git reset --hard %s " % new_rev
)
1204 # Update and stage the (potentially modified)
1205 # combo-layer.conf, but do not commit separately.
1207 repo
= conf
. repos
[ name
]
1208 rev
= repo
[ 'last_revision' ]
1209 conf
. update ( name
, "last_revision" , rev
)
1210 if commit_conf_file ( conf
, components
, False ):
1211 # Must augment the previous commit.
1212 runcmd ( "git commit --amend -C HEAD" )
1215 scanned_revs
= False
1216 def find_revs ( old2new
, head
):
1217 '''Construct mapping from original commit hash to commit hash in
1218 combined repo by looking at the commit messages. Depends on the
1219 "From ... rev: ..." convention.'''
1220 logger
. info ( "Analyzing log messages to find previously imported commits..." )
1221 num_known
= len ( old2new
)
1222 log
= runcmd ( "git log --grep='From .* rev: [a-fA-F0-9][a-fA-F0-9]*' --pretty=format:%H %x00% B %x00 " + head
). split ( chr ( 0 ))
1223 regex
= re
. compile ( r
'From .* rev: ([a-fA-F0-9]+)' )
1224 for new_rev
, body
in zip (*[ iter ( log
)]* 2 ):
1225 # Use the last one, in the unlikely case there are more than one.
1226 rev
= regex
. findall ( body
)[- 1 ]
1227 if rev
not in old2new
:
1228 old2new
[ rev
] = new_rev
. strip ()
1229 logger
. info ( "Found %d additional commits, leading to: %s " % ( len ( old2new
) - num_known
, old2new
))
1232 def apply_commit ( parent
, rev
, largs
, wargs
, dest_dir
, file_filter
= None ):
1233 '''Compare revision against parent, remove files deleted in the
1234 commit, re-write new or modified ones. Moves them into dest_dir.
1235 Optionally filters files.
1239 # -r recurses into sub-directories, given is the full overview of
1240 # what changed. We do not care about copy/edits or renames, so we
1241 # can disable those with --no-renames (but we still parse them,
1242 # because it was not clear from git documentation whether C and M
1243 # lines can still occur).
1244 logger
. debug ( "Applying changes between %s and %s in %s " % ( parent
, rev
, largs
[ "destdir" ]))
1249 changes
= runcmd ( "git diff-tree --no-commit-id --no-renames --name-status -r --raw -z %s %s " % ( parent
, rev
), ** largs
). split ( chr ( 0 ))
1250 for status
, name
in zip (*[ iter ( changes
)]* 2 ):
1251 if status
[ 0 ] in "ACMRT" :
1253 elif status
[ 0 ] in "D" :
1256 logger
. error ( "Unknown status %s of file %s in revision %s " % ( status
, name
, rev
))
1260 update
. extend ( runcmd ( "git ls-tree -r --name-only -z %s " % rev
, ** largs
). split ( chr ( 0 )))
1262 # Include/exclude files as define in the component config.
1263 # Both updated and deleted file lists get filtered, because it might happen
1264 # that a file gets excluded, pulled from a different component, and then the
1265 # excluded file gets deleted. In that case we must keep the copy.
1270 # We export into a tar archive here and extract with tar because it is simple (no
1271 # need to implement file and symlink writing ourselves) and gives us some degree
1272 # of parallel IO. The downside is that we have to pass the list of files via
1273 # command line parameters - hopefully there will never be too many at once.
1275 target
= os
. path
. join ( wargs
[ "destdir" ], dest_dir
)
1276 if not os
. path
. isdir ( target
):
1278 quoted_target
= shlex
. quote ( target
)
1279 # os.sysconf('SC_ARG_MAX') is lying: running a command with
1280 # string length 629343 already failed with "Argument list too
1281 # long" although SC_ARG_MAX = 2097152. "man execve" explains
1282 # the limitations, but those are pretty complicated. So here
1283 # we just hard-code a fixed value which is more likely to work.
1284 max_cmdsize
= 64 * 1024
1288 cmdsize
= 100 + len ( quoted_target
)
1290 quoted_next
= shlex
. quote ( update
[ 0 ])
1291 size_next
= len ( quoted_next
) + len ( dest_dir
) + 1
1292 logger
. debug ( 'cmdline length %d + %d < %d ?' % ( cmdsize
, size_next
, os
. sysconf ( 'SC_ARG_MAX' )))
1293 if cmdsize
+ size_next
< max_cmdsize
:
1294 quoted_args
. append ( quoted_next
)
1295 unquoted_args
. append ( update
. pop ( 0 ))
1296 cmdsize
+= size_next
1298 logger
. debug ( 'Breaking the cmdline at length %d ' % cmdsize
)
1300 logger
. debug ( 'Final cmdline length %d / %d ' % ( cmdsize
, os
. sysconf ( 'SC_ARG_MAX' )))
1301 cmd
= "git archive %s %s | tar -C %s -xf -" % ( rev
, ' ' . join ( quoted_args
), quoted_target
)
1302 logger
. debug ( 'First cmdline length %d ' % len ( cmd
))
1303 runcmd ( cmd
, ** largs
)
1304 cmd
= "git add -f" . split () + [ os
. path
. join ( dest_dir
, x
) for x
in unquoted_args
]
1305 logger
. debug ( 'Second cmdline length %d ' % reduce ( lambda x
, y
: x
+ len ( y
), cmd
, 0 ))
1306 runcmd ( cmd
, ** wargs
)
1310 path
= os
. path
. join ( dest_dir
, path
)
1311 runcmd ( "git rm -f --ignore-unmatch" . split () + [ os
. path
. join ( dest_dir
, x
) for x
in delete
], ** wargs
)
1313 def action_error ( conf
, args
):
1314 logger
. info ( "invalid action %s " % args
[ 0 ])
1317 "init" : action_init
,
1318 "update" : action_update
,
1319 "pull" : action_pull
,
1320 "splitpatch" : action_splitpatch
,
1321 "sync-revs" : action_sync_revs
,
1325 parser
= optparse
. OptionParser (
1326 version
= "Combo Layer Repo Tool version %s " % __version__
,
1327 usage
= """%prog [options] action
1329 Create and update a combination layer repository from multiple component repositories.
1332 init initialise the combo layer repo
1333 update [components] get patches from component repos and apply them to the combo repo
1334 pull [components] just pull component repos only
1335 sync-revs [components] update the config file's last_revision for each repository
1336 splitpatch [commit] generate commit patch and split per component, default commit is HEAD""" )
1338 parser
. add_option ( "-c" , "--conf" , help = "specify the config file (conf/combo-layer.conf is the default)." ,
1339 action
= "store" , dest
= "conffile" , default
= "conf/combo-layer.conf" )
1341 parser
. add_option ( "-i" , "--interactive" , help = "interactive mode, user can edit the patch list and patches" ,
1342 action
= "store_true" , dest
= "interactive" , default
= False )
1344 parser
. add_option ( "-D" , "--debug" , help = "output debug information" ,
1345 action
= "store_true" , dest
= "debug" , default
= False )
1347 parser
. add_option ( "-n" , "--no-pull" , help = "skip pulling component repos during update" ,
1348 action
= "store_true" , dest
= "nopull" , default
= False )
1350 parser
. add_option ( "--hard-reset" ,
1351 help = "instead of pull do fetch and hard-reset in component repos" ,
1352 action
= "store_true" , dest
= "hard_reset" , default
= False )
1354 parser
. add_option ( "-H" , "--history" , help = "import full history of components during init" ,
1355 action
= "store_true" , default
= False )
1357 options
, args
= parser
. parse_args ( sys
. argv
)
1359 # Dispatch to action handler
1361 logger
. error ( "No action specified, exiting" )
1363 elif args
[ 1 ] not in actions
:
1364 logger
. error ( "Unsupported action %s , exiting \n " % ( args
[ 1 ]))
1366 elif not os
. path
. exists ( options
. conffile
):
1367 logger
. error ( "No valid config file, exiting \n " )
1371 logger
. setLevel ( logging
. DEBUG
)
1372 confdata
= Configuration ( options
)
1373 initmode
= ( args
[ 1 ] == 'init' )
1374 confdata
. sanity_check ( initmode
)
1375 actions
. get ( args
[ 1 ], action_error
)( confdata
, args
[ 1 :])
1377 if __name__
== "__main__" :
1383 traceback
. print_exc ()