Skip to content

Commit

Permalink
Added artifact filters support to list file entries script
Browse files Browse the repository at this point in the history
  • Loading branch information
joachimmetz committed Jan 22, 2022
1 parent 045e584 commit 6cb3acf
Show file tree
Hide file tree
Showing 17 changed files with 1,172 additions and 55 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test_docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- name: Install dependencies
run: |
dnf copr -y enable @gift/dev
dnf install -y @development-tools python3 python3-devel libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dfvfs python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools python3-six
dnf install -y @development-tools python3 python3-devel libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi python3-cryptography python3-dfdatetime python3-dfvfs python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools python3-six
- name: Run tests
env:
LANG: C.utf8
Expand Down Expand Up @@ -57,7 +57,7 @@ jobs:
run: |
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential python3 python3-dev libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfvfs python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyxattr python3-setuptools python3-six python3-yaml
apt-get install -y build-essential python3 python3-dev libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfvfs python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyxattr python3-setuptools python3-six python3-yaml
- name: Run tests
env:
LANG: en_US.UTF-8
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_tox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
add-apt-repository -y ppa:deadsnakes/ppa
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfvfs python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-six python3-yaml
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfvfs python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-six python3-yaml
- name: Install tox
run: |
python3 -m pip install tox
Expand Down
2 changes: 1 addition & 1 deletion .pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code.
extension-pkg-allow-list=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pyluksde,pymodi,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm
extension-pkg-allow-list=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pyluksde,pymodi,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm,xattr

# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
Expand Down
2 changes: 1 addition & 1 deletion appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ environment:
HOMEBREW_NO_INSTALL_CLEANUP: 1

install:
- cmd: "%PYTHON%\\python.exe -m pip install -U pip setuptools wheel"
- cmd: "%PYTHON%\\python.exe -m pip install -U pip setuptools twine wheel"
- cmd: "%PYTHON%\\python.exe -m pip install pywin32 WMI"
- cmd: "%PYTHON%\\python.exe %PYTHON%\\Scripts\\pywin32_postinstall.py -install"
- ps: If ($isWindows) { .\config\appveyor\install.ps1 }
Expand Down
2 changes: 1 addition & 1 deletion config/appveyor/install.ps1
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Script to set up tests on AppVeyor Windows.

$Dependencies = "PyYAML cffi cryptography dfdatetime dfvfs dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libmodi libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six xattr"
$Dependencies = "PyYAML artifacts cffi cryptography dfdatetime dfvfs dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libmodi libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six xattr"
$Dependencies = ${Dependencies} -split " "

$Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1"
Expand Down
2 changes: 1 addition & 1 deletion config/dpkg/control
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfimagetools

Package: python3-dfimagetools
Architecture: all
Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20210721), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211225), python3-dfvfs (>= 20211227), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20210721), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20211205), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211225), python3-dfvfs (>= 20211227), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
Description: Python 3 module of dfImageTools
Collection of tools to process storage media images.

Expand Down
6 changes: 6 additions & 0 deletions dependencies.ini
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
[artifacts]
dpkg_name: python3-artifacts
minimum_version: 20211205
rpm_name: python3-artifacts
version_property: __version__

[cffi]
skip_check: true
dpkg_name: python3-cffi-backend
Expand Down
115 changes: 115 additions & 0 deletions dfimagetools/artifact_filters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
"""Helper for filtering based on artifact definitions."""

import logging

from artifacts import definitions as artifacts_definitions

from dfvfs.helpers import file_system_searcher as dfvfs_file_system_searcher

from dfimagetools import path_resolver


class ArtifactDefinitionFiltersGenerator(object):
"""Generator of filters based on artifact definitions."""

def __init__(self, artifacts_registry, environment_variables, user_accounts):
"""Initializes an artifact definition filters generator.
Args:
artifacts_registry (artifacts.ArtifactDefinitionsRegistry): artifact
definitions registry.
environment_variables (list[EnvironmentVariable]): environment variables.
user_accounts (list[UserAccount]): user accounts.
"""
super(ArtifactDefinitionFiltersGenerator, self).__init__()
self._artifacts_registry = artifacts_registry
self._environment_variables = environment_variables
self._path_resolver = path_resolver.PathResolver()
self._user_accounts = user_accounts

def _BuildFindSpecsFromArtifactDefinition(self, name):
"""Builds find specifications from an artifact definition.
Args:
name (str): name of the artifact definition.
Yields:
dfvfs.FindSpec: file system (dfVFS) find specification.
"""
definition = self._artifacts_registry.GetDefinitionByName(name)
if not definition:
logging.warning('Undefined artifact definition: {0:s}'.format(name))
else:
for source in definition.sources:
source_type = source.type_indicator
if source_type not in (
artifacts_definitions.TYPE_INDICATOR_ARTIFACT_GROUP,
artifacts_definitions.TYPE_INDICATOR_FILE):
logging.warning((
'Unsupported source type: {0:s} in artifact definition: '
'{1:s}"').format(source_type, name))
continue

if source_type == artifacts_definitions.TYPE_INDICATOR_ARTIFACT_GROUP:
for source_name in set(source.names):
for find_spec in self._BuildFindSpecsFromArtifactDefinition(
source_name):
yield find_spec

elif source_type == artifacts_definitions.TYPE_INDICATOR_FILE:
for source_path in set(source.paths):
for find_spec in self._BuildFindSpecsFromFileSourcePath(
source_path, source.separator):
yield find_spec

def _BuildFindSpecsFromFileSourcePath(self, source_path, path_separator):
"""Builds find specifications from a file source type.
Args:
source_path (str): file system path defined by the source.
path_separator (str): file system path segment separator.
Yields:
dfvfs.FindSpec: file system (dfVFS) find specification.
"""
for path_glob in self._path_resolver.ExpandGlobStars(
source_path, path_separator):

for path in self._path_resolver.ExpandUsersVariable(
path_glob, path_separator, self._user_accounts):

if '%' in path:
path = self._path_resolver.ExpandEnvironmentVariables(
path, path_separator, self._environment_variables)

if not path.startswith(path_separator):
logging.warning((
'The path filter must be defined as an absolute path: '
'"{0:s}"').format(path))
continue

try:
find_spec = dfvfs_file_system_searcher.FindSpec(
case_sensitive=False, location_glob=path,
location_separator=path_separator)
except ValueError as exception:
logging.error((
'Unable to build find specification for path: "{0:s}" with '
'error: {1!s}').format(path, exception))
continue

yield find_spec

def GetFindSpecs(self, names):
"""Retrieves find specifications based on artifact definitions.
Args:
names (list[str]): names of the artifact definitions to filter on.
Yields:
dfvfs.FindSpec: file system (dfVFS) find specification.
"""
for name in set(names):
for find_spec in self._BuildFindSpecsFromArtifactDefinition(name):
yield find_spec
114 changes: 76 additions & 38 deletions dfimagetools/file_entry_lister.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,17 @@
import logging
import re

from dfvfs.analyzer import analyzer
from dfvfs.analyzer import fvde_analyzer_helper
from dfvfs.helpers import file_system_searcher
from dfvfs.helpers import volume_scanner
from dfvfs.lib import definitions as dfvfs_definitions
from dfvfs.resolver import resolver
from dfvfs.path import factory as dfvfs_path_spec_factory
from dfvfs.resolver import resolver as dfvfs_resolver
from dfvfs.volume import factory as dfvfs_volume_system_factory

from dfimagetools import bodyfile
from dfimagetools import decorators


try:
# Disable experimental FVDE support.
analyzer.Analyzer.DeregisterHelper(fvde_analyzer_helper.FVDEAnalyzerHelper())
except KeyError:
pass


class FileEntryLister(volume_scanner.VolumeScanner):
"""File entry lister."""

Expand All @@ -37,6 +30,47 @@ def __init__(self, mediator=None):
self._bodyfile_generator = bodyfile.BodyfileGenerator()
self._list_only_files = False

def _GetBasePathSegments(self, base_path_spec):
"""Retrieves the base path segments.
Args:
base_path_specs (list[dfvfs.PathSpec]): source path specification.
Returns:
list[str]: path segments.
"""
if not base_path_spec.HasParent() or not base_path_spec.parent:
return ['']

if base_path_spec.parent.type_indicator in (
dfvfs_definitions.TYPE_INDICATOR_APFS_CONTAINER,
dfvfs_definitions.TYPE_INDICATOR_GPT,
dfvfs_definitions.TYPE_INDICATOR_LVM):
volume_system = dfvfs_volume_system_factory.Factory.NewVolumeSystem(
base_path_spec.parent.type_indicator)
volume_system.Open(base_path_spec.parent)

volume = volume_system.GetVolumeByIdentifier(
base_path_spec.parent.location[1:])

if base_path_spec.parent.type_indicator == (
dfvfs_definitions.TYPE_INDICATOR_GPT):
volume_identifier_prefix = 'gpt'
else:
volume_identifier_prefix = volume_system.VOLUME_IDENTIFIER_PREFIX

volume_identifier = volume.GetAttribute('identifier')

volume_path_segment = '{0:s}{{{1:s}}}'.format(
volume_identifier_prefix, volume_identifier.value)
return ['', volume_path_segment]

if base_path_spec.parent.type_indicator == (
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION):
return base_path_spec.parent.location.split('/')

return ['']

def _GetPathSpecificationString(self, path_spec):
"""Retrieves a printable string representation of the path specification.
Expand Down Expand Up @@ -97,49 +131,53 @@ def ListFileEntries(self, base_path_specs):
"""Lists file entries in the base path specification.
Args:
base_path_specs (list[dfvfs.PathSpec]): source path specification.
base_path_specs (list[dfvfs.PathSpec]): source path specifications.
Yields:
tuple[dfvfs.FileEntry, list[str]]: file entry and path segments.
"""
for base_path_spec in base_path_specs:
file_system = resolver.Resolver.OpenFileSystem(base_path_spec)
file_entry = resolver.Resolver.OpenFileEntry(base_path_spec)
file_system = dfvfs_resolver.Resolver.OpenFileSystem(base_path_spec)
file_entry = dfvfs_resolver.Resolver.OpenFileEntry(base_path_spec)
if file_entry is None:
path_specification_string = self._GetPathSpecificationString(
base_path_spec)
logging.warning('Unable to open base path specification:\n{0:s}'.format(
path_specification_string))
return

base_path_segments = ['']
if base_path_spec.HasParent() and base_path_spec.parent:
if base_path_spec.parent.type_indicator in (
dfvfs_definitions.TYPE_INDICATOR_APFS_CONTAINER,
dfvfs_definitions.TYPE_INDICATOR_GPT,
dfvfs_definitions.TYPE_INDICATOR_LVM):
volume_system = dfvfs_volume_system_factory.Factory.NewVolumeSystem(
base_path_spec.parent.type_indicator)
volume_system.Open(base_path_spec.parent)
base_path_segments = self._GetBasePathSegments(base_path_spec)
for result in self._ListFileEntry(
file_system, file_entry, base_path_segments):
yield result

volume = volume_system.GetVolumeByIdentifier(
base_path_spec.parent.location[1:])
def ListFileEntriesWithFindSpecs(self, base_path_specs, find_specs):
"""Lists file entries in the base path specification.
if base_path_spec.parent.type_indicator == (
dfvfs_definitions.TYPE_INDICATOR_GPT):
volume_identifier_prefix = 'gpt'
else:
volume_identifier_prefix = volume_system.VOLUME_IDENTIFIER_PREFIX
Args:
base_path_specs (list[dfvfs.PathSpec]): source path specification.
find_specs (list[dfvfs.FindSpec]): find specifications.
volume_identifier = volume.GetAttribute('identifier')
Yields:
tuple[dfvfs.FileEntry, list[str]]: file entry and path segments.
"""
for base_path_spec in base_path_specs:
file_system = dfvfs_resolver.Resolver.OpenFileSystem(base_path_spec)

base_path_segments = ['', '{0:s}{{{1:s}}}'.format(
volume_identifier_prefix, volume_identifier.value)]
if dfvfs_path_spec_factory.Factory.IsSystemLevelTypeIndicator(
base_path_spec.type_indicator):
mount_point = base_path_spec
else:
mount_point = base_path_spec.parent

elif base_path_spec.parent.type_indicator == (
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION):
base_path_segments = base_path_spec.parent.location.split('/')
base_path_segments = self._GetBasePathSegments(base_path_spec)

for result in self._ListFileEntry(
file_system, file_entry, base_path_segments):
yield result
searcher = file_system_searcher.FileSystemSearcher(
file_system, mount_point)
for path_spec in searcher.Find(find_specs=find_specs):
file_entry = dfvfs_resolver.Resolver.OpenFileEntry(path_spec)
path_segments = file_system.SplitPath(path_spec.location)

full_path_segments = list(base_path_segments)
full_path_segments.extend(path_segments)
yield file_entry, full_path_segments
Loading

0 comments on commit 6cb3acf

Please sign in to comment.