dependency_handler.py

`ofrak.service.dependency_handler`

`DependencyHandler`

Stateless handler for dealing with creating and invalidating dependencies. Intended for one-time use in a component, or possible re-use in a resource.

`create_component_dependencies(self, component_id, component_version)`

Register dependencies between the component and the resources it interacts with.

This may not even be necessary since Resource.add_attributes does this anyway...

Source code in ofrak/service/dependency_handler.py

def create_component_dependencies(
    self,
    component_id: bytes,
    component_version: int,
):
    """
    Register dependencies between the component and the resources it interacts with.

    This may not even be necessary since Resource.add_attributes does this anyway...
    """
    self._validate_resource_context_complete()

    for resource_id in self._component_context.resources_created:
        new_resource_m = self._resource_context.resource_models[resource_id]
        for attributes in new_resource_m.attributes.keys():
            new_resource_m.add_component_for_attributes(
                component_id, component_version, attributes
            )

`create_resource_dependencies(self, component_id)`

Register dependencies between a resource with some attributes and the resources which were accessed in the context where these attributes were added.

When a component runs, this method is called to record what data was accessed by the component and what resource attributes from other resources were accessed within that component. These registered dependencies allow for OFRAK to not rerun analyzers when the resource and its dependencies have not changed.

Whenever a Modifier is run, these resource attribute dependencies are invalidated so as to force analysis to be rerun.

Parameters:

Name	Type	Description	Default
`component_id`	`bytes`		required

Source code in ofrak/service/dependency_handler.py

def create_resource_dependencies(
    self,
    component_id: bytes,
):
    """
    Register dependencies between a resource with some attributes and the resources which
    were accessed in the context where these attributes were added.

    When a component runs, this method is called to record what data was accessed by the
    component and what resource attributes from other resources were accessed within that
    component. These registered dependencies allow for OFRAK to not rerun analyzers when the
    resource and its dependencies have not changed.

    Whenever a [Modifier][ofrak.component.modifier.Modifier] is run, these resource attribute
    dependencies are invalidated so as to force analysis to be rerun.

    :param bytes component_id:
    """
    self._validate_resource_context_complete()
    resource_dependencies = []

    # Create dependency for each attribute on newly created resources
    for resource_id in self._component_context.resources_created:
        new_resource_m = self._resource_context.resource_models[resource_id]
        for attributes in new_resource_m.attributes.keys():
            resource_dependencies.append(
                ResourceAttributeDependency(
                    resource_id,
                    component_id,
                    attributes,
                )
            )

    # Create dependency for each new attribute on modified resources
    for resource_id in self._component_context.modification_trackers.keys():
        modified_resource_m = self._resource_context.resource_models[resource_id]
        for attrs_added in modified_resource_m.diff.attributes_added.keys():
            resource_dependencies.append(
                ResourceAttributeDependency(
                    resource_id,
                    component_id,
                    attrs_added,
                )
            )

    # Add dependencies to all resources which were accessed
    for resource_id, access_tracker in self._component_context.access_trackers.items():
        if resource_id in self._component_context.resources_created:
            # Avoid all the newly created components depending on each other
            continue
        accessed_resource_m = self._resource_context.resource_models[resource_id]
        merged_accessed_ranges = Range.merge_ranges(access_tracker.data_accessed)
        # Add attributes dependency on all accessed attributes
        for attributes_accessed in access_tracker.attributes_accessed:
            for resource_dependency in resource_dependencies:
                accessed_resource_m.add_attribute_dependency(
                    attributes_accessed, resource_dependency
                )
        # Add data dependency on all accessed data
        for accessed_range in merged_accessed_ranges:
            for resource_dependency in resource_dependencies:
                accessed_resource_m.add_data_dependency(resource_dependency, accessed_range)

`_invalidate_dependencies(self, handled_dependencies, unhandled_dependencies, resources_modified)` `async` `private`

Invalidate the unhandled resource attribute dependencies.

Parameters:

Name	Type	Description	Default
`handled_dependencies`	`Set[ofrak.model.resource_model.ResourceAttributeDependency]`	A set of dependencies that have already been invalidated	required
`unhandled_dependencies`	`Set[ofrak.model.resource_model.ResourceAttributeDependency]`	A set of dependencies that should be invalidated	required

Source code in ofrak/service/dependency_handler.py

async def _invalidate_dependencies(
    self,
    handled_dependencies: Set[ResourceAttributeDependency],
    unhandled_dependencies: Set[ResourceAttributeDependency],
    resources_modified: Set[MutableResourceModel],
):
    """
    Invalidate the unhandled resource attribute dependencies.

    :param Set[ResourceAttributeDependency] handled_dependencies: A set of dependencies that
    have already been invalidated
    :param Set[ResourceAttributeDependency] unhandled_dependencies: A set of dependencies
    that should be invalidated
    """
    if len(unhandled_dependencies) == 0:
        return

    dependent_resource_ids = {
        dependency.dependent_resource_id for dependency in unhandled_dependencies
    }

    deleted_dependent_ids = {
        r_id
        for r_id, currently_exists in zip(
            dependent_resource_ids,
            await self._resource_service.verify_ids_exist(dependent_resource_ids),
        )
        if not currently_exists
    }
    await self._fetch_missing_resources(
        dependent_resource_ids.difference(deleted_dependent_ids)
    )

    # Invalidate the resources' attributes referred to by the unhandled_dependencies
    next_unhandled_dependencies = set()
    for dependency in unhandled_dependencies:
        # It's possible that the attribute was already invalidated from an earlier run
        if dependency in handled_dependencies:
            continue

        # If the dependent resource was deleted, don't need to propagate dependency invalidation
        if dependency.dependent_resource_id in deleted_dependent_ids:
            handled_dependencies.add(dependency)
            continue

        try:
            resource_m = self._resource_context.resource_models[
                dependency.dependent_resource_id
            ]
        except KeyError as e:
            missing_model = await self._resource_service.get_by_id(
                dependency.dependent_resource_id
            )
            resource_m = MutableResourceModel.from_model(missing_model)

        # Invalidate the attributes on the resource
        handled_dependencies.add(dependency)

        # The component id is not necessarily present. It could have been invalidated already
        # by a previous patch that impacted other resources that this resource depends on.
        if resource_m.get_component_id_by_attributes(dependency.attributes):
            resource_m.remove_component(dependency.component_id, dependency.attributes)
            self._component_context.mark_resource_modified(resource_m.id)
            resources_modified.add(resource_m)

        # Find other dependencies to invalidate due to the invalidation of the attributes
        invalidated_dependencies = set()
        for next_dependency in resource_m.attribute_dependencies[dependency.attributes]:
            # Make sure the dependency wasn't already handled
            if next_dependency not in handled_dependencies:
                LOGGER.debug(
                    f"Invalidating attributes {next_dependency.attributes.__name__} from "
                    f"component {next_dependency.component_id!r} on resource "
                    f"{next_dependency.dependent_resource_id.hex()} due to "
                    f"attributes {dependency.attributes.__name__} on resource"
                    f" {dependency.dependent_resource_id.hex()} being invalidated"
                )

                invalidated_dependencies.add(next_dependency)

        for invalidated_dependency in invalidated_dependencies:
            resource_m.remove_dependency(invalidated_dependency)
            self._component_context.mark_resource_modified(resource_m.id)
            resources_modified.add(resource_m)
        next_unhandled_dependencies.update(invalidated_dependencies)

    await self._invalidate_dependencies(
        handled_dependencies,
        next_unhandled_dependencies,
        resources_modified,
    )

dependency_handler.py

ofrak.service.dependency_handler