django.setup()
from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact
+from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile
from orm.models import Variable, VariableHistory
from orm.models import Package, Package_File, Target_Installed_Package, Target_File
from orm.models import Task_Dependency, Package_Dependency
return vars(self)[dictname][key]
+ def get_similar_target_with_image_files(self, target):
+ """
+ Get a Target object "similar" to target; i.e. with the same target
+ name ('core-image-minimal' etc.) and machine.
+ """
+ return target.get_similar_target_with_image_files()
+
def _timestamp_to_datetime(self, secs):
"""
Convert timestamp in seconds to Python datetime
file_name = file_name,
file_size = file_size)
- def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
+ def save_target_artifact_file(self, target_obj, file_name, file_size):
"""
- Save artifact information without checking for duplicate paths;
- this is used when we are saving data about an artifact which was
- generated by a previous build but which is also relevant to this build,
- e.g. a bzImage file.
+ Save artifact file information for a Target target_obj.
+
+ Note that this doesn't include SDK artifacts, only images and
+ related files (e.g. bzImage).
"""
- BuildArtifact.objects.create(build=build_obj, file_name=file_name,
- file_size=file_size)
+ TargetArtifactFile.objects.create(target=target_obj,
+ file_name=file_name, file_size=file_size)
def save_artifact_information(self, build_obj, file_name, file_size):
- # we skip the image files from other builds
- if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
- return
-
+ """
+ TODO this is currently used to save SDK artifacts to the database,
+ but will be replaced in future once SDK artifacts are associated with
+ Target objects (as they eventually should be)
+ """
# do not update artifacts found in other builds
if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
return
- self.save_artifact_information_no_dedupe(self, build_obj, file_name,
- file_size)
+ # do not update artifact if already a target artifact file for that path
+ if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
+ return
+
+ BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+ file_size=file_size)
def create_logmessage(self, log_information):
assert 'build' in log_information
self.orm_wrapper.create_logmessage(log_information)
- def _get_files_from_image_license(self, image_license_manifest_path):
+ def _get_filenames_from_image_license(self, image_license_manifest_path):
"""
Find the FILES line in the image_license.manifest file,
which has the basenames of the bzImage and modules files
OR
- 2. There are no files for the target, so copy them from a
- previous build with the same target + machine.
+ 2. There are no new files for the target (they were already produced by
+ a previous build), so copy them from the most recent previous build with
+ the same target, task and machine.
"""
deploy_dir_image = \
self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
# if there's no DEPLOY_DIR_IMAGE, there aren't going to be
- # any build artifacts, so we can return immediately
+ # any image artifacts, so we can return immediately
if not deploy_dir_image:
return
buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
- machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
+ machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
# location of the image_license.manifest files for this build;
image_file_extensions_unique = set(image_file_extensions.split(' '))
targets = self.internal_state['targets']
+
+ # filter out anything which isn't an image target
image_targets = [target for target in targets if target.is_image]
+
for target in image_targets:
# this is set to True if we find at least one file relating to
# this target; if this remains False after the scan, we copy the
if os.path.isfile(image_license_manifest_path):
has_files = True
- basenames = self._get_files_from_image_license(
+ basenames = self._get_filenames_from_image_license(
image_license_manifest_path)
for basename in basenames:
artifact_path = os.path.join(deploy_dir_image, basename)
artifact_size = os.stat(artifact_path).st_size
- self.orm_wrapper.save_artifact_information_no_dedupe(
- self.internal_state['build'], artifact_path,
- artifact_size)
+ # note that the artifact will only be saved against this
+ # build if it hasn't been already
+ self.orm_wrapper.save_target_artifact_file(target,
+ artifact_path, artifact_size)
# store the license manifest path on the target
# (this file is also created any time an image file is created)
# (via real_image_name); note that we don't have to set
# has_files = True, as searching for the license manifest file
# will already have set it to true if at least one image file was
- # produced
+ # produced; note that the real_image_name includes BUILDNAME, which
+ # in turn includes a timestamp; so if no files were produced for
+ # this timestamp (i.e. the build reused existing image files already
+ # in the directory), no files will be recorded against this target
image_files = self._get_image_files(deploy_dir_image,
real_image_name, image_file_extensions_unique)
target, image_file['path'], image_file['size'])
if not has_files:
- # TODO copy artifact and image files from the
- # most-recently-built Target with the same target + machine
- # as this Target; also copy the license manifest path,
- # as that is treated differently
- pass
+ # copy image files and build artifacts from the
+ # most-recently-built Target with the
+ # same target + machine as this Target; also copy the license
+ # manifest path, as that is not treated as an artifact and needs
+ # to be set separately
+ most_recent = \
+ self.orm_wrapper.get_similar_target_with_image_files(target)
+
+ if most_recent:
+ logger.info('image artifacts for target %s cloned from ' \
+ 'target %s' % (target.pk, most_recent.pk))
+ target.clone_artifacts_from(most_recent)
def close(self, errorcode):
if self.brbe is not None:
--- /dev/null
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0007_auto_20160523_1446'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='TargetArtifactFile',
+ fields=[
+ ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
+ ('file_name', models.FilePathField()),
+ ('file_size', models.IntegerField()),
+ ('target', models.ForeignKey(to='orm.Target')),
+ ],
+ ),
+ ]
from __future__ import unicode_literals
from django.db import models, IntegrityError
-from django.db.models import F, Q, Avg, Max, Sum
+from django.db.models import F, Q, Avg, Max, Sum, Count
from django.utils import timezone
from django.utils.encoding import force_bytes
def get_image_file_extensions(self):
"""
- Get list of file name extensions for images produced by this build
+ Get list of file name extensions for images produced by this build;
+ note that this is the actual list of extensions stored on Target objects
+ for this build, and not the value of IMAGE_FSTYPES.
"""
extensions = []
return ', '.join(extensions)
+ def get_image_fstypes(self):
+ """
+ Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
+ list of image file suffixes.
+ """
+ image_fstypes = Variable.objects.get(
+ build=self, variable_name='IMAGE_FSTYPES').variable_value
+ return list(set(re.split(r' {1,}', image_fstypes)))
+
def get_sorted_target_list(self):
tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
return( tgts );
def __unicode__(self):
return self.target
+ def get_similar_targets(self):
+ """
+ Get targets for the same machine, task and target name
+ (e.g. 'core-image-minimal') from a successful build for this project
+ (but excluding this target).
+
+ Note that we look for targets built by this project because projects
+ can have different configurations from each other, and put their
+ artifacts in different directories.
+ """
+ query = ~Q(pk=self.pk) & \
+ Q(target=self.target) & \
+ Q(build__machine=self.build.machine) & \
+ Q(build__outcome=Build.SUCCEEDED) & \
+ Q(build__project=self.build.project)
+
+ return Target.objects.filter(query)
+
+ def get_similar_target_with_image_files(self):
+ """
+ Get the most recent similar target with Target_Image_Files associated
+ with it, for the purpose of cloning those files onto this target.
+ """
+ similar_target = None
+
+ candidates = self.get_similar_targets()
+ if candidates.count() < 1:
+ return similar_target
+
+ task_subquery = Q(task=self.task)
+
+ # we can look for a 'build' task if this task is a 'populate_sdk_ext'
+ # task, as it will have created images; and vice versa; note that
+ # 'build' targets can have their task set to '';
+ # also note that 'populate_sdk' does not produce image files
+ image_tasks = [
+ '', # aka 'build'
+ 'build',
+ 'populate_sdk_ext'
+ ]
+ if self.task in image_tasks:
+ task_subquery = Q(task__in=image_tasks)
+
+ query = task_subquery & Q(num_files__gt=0)
+
+ # annotate with the count of files, to exclude any targets which
+ # don't have associated files
+ candidates = candidates.annotate(
+ num_files=Count('target_image_file'))
+
+ candidates = candidates.filter(query)
+
+ if candidates.count() > 0:
+ candidates.order_by('build__completed_on')
+ similar_target = candidates.last()
+
+ return similar_target
+
+ def clone_artifacts_from(self, target):
+ """
+ Make clones of the BuildArtifacts, Target_Image_Files and
+ TargetArtifactFile objects associated with Target target, then
+ associate them with this target.
+
+ Note that for Target_Image_Files, we only want files from the previous
+ build whose suffix matches one of the suffixes defined in this
+ target's build's IMAGE_FSTYPES configuration variable. This prevents the
+ Target_Image_File object for an ext4 image being associated with a
+ target for a project which didn't produce an ext4 image (for example).
+
+ Also sets the license_manifest_path of this target to the same path
+ as that of target being cloned from, as the license manifest path is
+ also a build artifact but is treated differently.
+ """
+
+ image_fstypes = self.build.get_image_fstypes()
+
+ # filter out any image files whose suffixes aren't in the
+ # IMAGE_FSTYPES suffixes variable for this target's build
+ image_files = [target_image_file \
+ for target_image_file in target.target_image_file_set.all() \
+ if target_image_file.suffix in image_fstypes]
+
+ for image_file in image_files:
+ image_file.pk = None
+ image_file.target = self
+ image_file.save()
+
+ artifact_files = target.targetartifactfile_set.all()
+ for artifact_file in artifact_files:
+ artifact_file.pk = None
+ artifact_file.target = self
+ artifact_file.save()
+
+ self.license_manifest_path = target.license_manifest_path
+ self.save()
+
+# an Artifact is anything that results from a target being built, and may
+# be of interest to the user, and is not an image file
+class TargetArtifactFile(models.Model):
+ target = models.ForeignKey(Target)
+ file_name = models.FilePathField()
+ file_size = models.IntegerField()
+
+ @property
+ def basename(self):
+ return os.path.basename(self.file_name)
+
class Target_Image_File(models.Model):
# valid suffixes for image files produced by a build
SUFFIXES = {
{% for target in targets %}
{% if target.target.is_image %}
<div class="well well-transparent dashboard-section">
- <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target}}</a></h3>
+ <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target.target}}</a></h3>
<dl class="dl-horizontal">
<dt>Packages included</dt>
<dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>
{% endfor %}
</ul>
</dd>
+ <dt>
+ Kernel artifacts
+ </dt>
+ <dd>
+ <ul class="list-unstyled">
+ {% for artifact in target.target_artifacts|dictsort:"basename" %}
+ <li>
+ <a href="{% url 'build_artifact' build.id 'targetartifactfile' artifact.id %}">{{artifact.basename}}</a>
+ ({{artifact.file_size|filtered_filesizeformat}})
+ </li>
+ {% endfor %}
+ </ul>
+ </dd>
</dl>
{% endif %}
</div>
from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage
+from orm.models import TargetArtifactFile
from orm.models import BitbakeVersion, CustomImageRecipe
from bldcontrol import bbcontroller
from django.views.decorators.cache import cache_control
targetHasNoImages = True
elem[ 'imageFiles' ] = imageFiles
elem[ 'targetHasNoImages' ] = targetHasNoImages
+ elem['target_artifacts'] = t.targetartifactfile_set.all()
+
targets.append( elem )
##
elif artifact_type == "buildartifact":
file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name
+ elif artifact_type == "targetartifactfile":
+ target = TargetArtifactFile.objects.get(pk=artifact_id)
+ file_name = target.file_name
+
elif artifact_type == "licensemanifest":
file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path
projects / thirdparty / openembedded / openembedded-core-contrib.git / commitdiff
