diff --git a/config/dpkg/control b/config/dpkg/control index 4088e9f36b..6627c738c3 100644 --- a/config/dpkg/control +++ b/config/dpkg/control @@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline) Package: python3-plaso Architecture: all -Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} +Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200613), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} Description: Python 3 module of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical diff --git a/config/linux/ubuntu_install_plaso.sh b/config/linux/ubuntu_install_plaso.sh index d61df73d1b..52a0b5f973 100755 --- a/config/linux/ubuntu_install_plaso.sh +++ b/config/linux/ubuntu_install_plaso.sh @@ -41,7 +41,7 @@ PYTHON_DEPENDENCIES="libbde-python3 python3-bencode python3-biplist python3-certifi - python3-cffi + python3-cffi-backend python3-chardet python3-cryptography python3-dateutil diff --git a/config/travis/install.sh b/config/travis/install.sh index 6b7c038b28..6a9493559b 100755 --- a/config/travis/install.sh +++ b/config/travis/install.sh @@ -5,7 +5,7 @@ # This file is generated by l2tdevtools update-dependencies.py any dependency # related changes should be made in dependencies.ini. -DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libluksde-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-cffi python3-chardet python3-cryptography python3-dateutil python3-defusedxml python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-elasticsearch python3-future python3-idna python3-lz4 python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-redis python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq"; +DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libluksde-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-cffi-backend python3-chardet python3-cryptography python3-dateutil python3-defusedxml python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-elasticsearch python3-future python3-idna python3-lz4 python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-redis python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq"; DPKG_PYTHON3_TEST_DEPENDENCIES="python3-coverage python3-distutils python3-fakeredis python3-mock python3-pbr python3-setuptools python3-sortedcontainers"; diff --git a/dependencies.ini b/dependencies.ini index 56b9083420..960e17cf5c 100644 --- a/dependencies.ini +++ b/dependencies.ini @@ -55,7 +55,7 @@ version_property: __version__ [dfdatetime] dpkg_name: python3-dfdatetime -minimum_version: 20200501 +minimum_version: 20200613 rpm_name: python3-dfdatetime version_property: __version__ diff --git a/plaso/cli/psort_tool.py b/plaso/cli/psort_tool.py index ed54d92bf6..aa8443b04f 100644 --- a/plaso/cli/psort_tool.py +++ b/plaso/cli/psort_tool.py @@ -32,6 +32,8 @@ from plaso.multi_processing import psort from plaso.storage import factory as storage_factory +import pytz # pylint: disable=wrong-import-order + class PsortTool( tools.CLITool, diff --git a/plaso/dependencies.py b/plaso/dependencies.py index 4a569d9ee7..6de2a2f211 100644 --- a/plaso/dependencies.py +++ b/plaso/dependencies.py @@ -27,7 +27,7 @@ 'cryptography': ('__version__', '2.0.2', None, True), 'dateutil': ('__version__', '1.5', None, True), 'defusedxml': ('__version__', '0.5.0', None, True), - 'dfdatetime': ('__version__', '20200501', None, True), + 'dfdatetime': ('__version__', '20200613', None, True), 'dfvfs': ('__version__', '20200604', None, True), 'dfwinreg': ('__version__', '20180712', None, True), 'dtfabric': ('__version__', '20181128', None, True), diff --git a/plaso/lib/timelib.py b/plaso/lib/timelib.py index 1b31fa0f01..29ec3dfe99 100644 --- a/plaso/lib/timelib.py +++ b/plaso/lib/timelib.py @@ -15,11 +15,9 @@ import datetime import logging -import dateutil.parser import pytz from plaso.lib import definitions -from plaso.lib import errors # pylint: disable=missing-type-doc,missing-return-type-doc @@ -239,55 +237,6 @@ def CopyToIsoFormat(cls, timestamp, timezone=pytz.UTC, raise_error=False): return datetime_object.isoformat() - @classmethod - def FromTimeString( - cls, time_string, dayfirst=False, gmt_as_timezone=True, - timezone=pytz.UTC): - """Converts a string containing a date and time value into a timestamp. - - Args: - time_string: String that contains a date and time value. - dayfirst: An optional boolean argument. If set to true then the - parser will change the precedence in which it parses timestamps - from MM-DD-YYYY to DD-MM-YYYY (and YYYY-MM-DD will be - YYYY-DD-MM, etc). - gmt_as_timezone: Sometimes the dateutil parser will interpret GMT and UTC - the same way, that is not make a distinction. By default - this is set to true, that is GMT can be interpreted - differently than UTC. If that is not the expected result - this attribute can be set to false. - timezone: Optional timezone object (instance of pytz.timezone) that - the data and time value in the string represents. This value - is used when the timezone cannot be determined from the string. - - Returns: - The timestamp which is an integer containing the number of microseconds - since January 1, 1970, 00:00:00 UTC or 0 on error. - - Raises: - TimestampError: if the time string could not be parsed. - """ - if not gmt_as_timezone and time_string.endswith(' GMT'): - time_string = '{0:s}UTC'.format(time_string[:-3]) - - try: - # TODO: deprecate the use of dateutil parser. - datetime_object = dateutil.parser.parse(time_string, dayfirst=dayfirst) - - except (TypeError, ValueError) as exception: - raise errors.TimestampError(( - 'Unable to convert time string: {0:s} in to a datetime object ' - 'with error: {1!s}').format(time_string, exception)) - - if datetime_object.tzinfo: - datetime_object = datetime_object.astimezone(pytz.UTC) - else: - datetime_object = timezone.localize(datetime_object) - - posix_time = int(calendar.timegm(datetime_object.utctimetuple())) - timestamp = posix_time * definitions.MICROSECONDS_PER_SECOND - return timestamp + datetime_object.microsecond - @classmethod def LocaltimeToUTC(cls, timestamp, timezone, is_dst=False): """Converts the timestamp in localtime of the timezone to UTC. diff --git a/plaso/parsers/java_idx.py b/plaso/parsers/java_idx.py index eb6e6a5889..433f509adb 100644 --- a/plaso/parsers/java_idx.py +++ b/plaso/parsers/java_idx.py @@ -10,12 +10,13 @@ # be present. 6.02 files will currently return 'Unknown'. from dfdatetime import java_time as dfdatetime_java_time +from dfdatetime import semantic_time as dfdatetime_semantic_time +from dfdatetime import time_elements as dfdatetime_time_elements from plaso.containers import events from plaso.containers import time_events from plaso.lib import errors from plaso.lib import definitions -from plaso.lib import timelib from plaso.parsers import dtfabric_parser from plaso.parsers import manager @@ -158,22 +159,23 @@ def ParseFileObject(self, parser_mediator, file_object): parser_mediator.ProduceEventWithEventData(event, event_data) if date_http_header: - # A HTTP header date and string "should" be in UTC or have an associated - # time zone information in the string itself. If that is not the case - # then there is no reliable method for plaso to determine the proper - # time zone, so the assumption is that it is UTC. + # A HTTP header date and time should be formatted according to RFC 1123. + # The date "should" be in UTC or have associated time zone information + # in the string itself. If that is not the case then there is no reliable + # method for Plaso to determine the proper time zone, so the assumption + # is that the date and time is in UTC. try: - download_date = timelib.Timestamp.FromTimeString( - date_http_header.value, gmt_as_timezone=False) - except errors.TimestampError: - parser_mediator.ProduceExtractionWarning( - 'Unable to parse date HTTP header value: {0:s}'.format( - date_http_header.value)) + date_time = dfdatetime_time_elements.TimeElements() + date_time.CopyFromStringRFC1123(date_http_header.value) + except ValueError as exception: + parser_mediator.ProduceExtractionWarning(( + 'Unable to parse date HTTP header string: {0:s} with error: ' + '{1!s}').format(date_http_header.value, exception)) + date_time = dfdatetime_semantic_time.InvalidTime() - if download_date: - event = time_events.TimestampEvent( - download_date, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED) - parser_mediator.ProduceEventWithEventData(event, event_data) + event = time_events.DateTimeValuesEvent( + date_time, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED) + parser_mediator.ProduceEventWithEventData(event, event_data) manager.ParsersManager.RegisterParser(JavaIDXParser) diff --git a/requirements.txt b/requirements.txt index d7b47b6c1f..add4c9c251 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ cffi >= 1.9.1 chardet >= 2.0.1 cryptography >= 2.0.2 defusedxml >= 0.5.0 -dfdatetime >= 20180704 +dfdatetime >= 20200613 dfvfs >= 20200604 dfwinreg >= 20180712 dtfabric >= 20181128 diff --git a/setup.cfg b/setup.cfg index e716639bb9..183f20dda2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ requires = libbde-python3 >= 20140531 python3-cryptography >= 2.0.2 python3-dateutil >= 1.5 python3-defusedxml >= 0.5.0 - python3-dfdatetime >= 20180704 + python3-dfdatetime >= 20200613 python3-dfvfs >= 20200604 python3-dfwinreg >= 20180712 python3-dtfabric >= 20181128 diff --git a/tests/lib/timelib.py b/tests/lib/timelib.py index 3d39023230..ea7ee486ac 100644 --- a/tests/lib/timelib.py +++ b/tests/lib/timelib.py @@ -6,7 +6,6 @@ import unittest -from plaso.lib import errors from plaso.lib import timelib import pytz # pylint: disable=wrong-import-order @@ -203,100 +202,6 @@ def testLocaltimeToUTC(self): '2013-01-01 05:00:00') self.assertEqual(timestamp, expected_timestamp) - def testTimestampFromTimeString(self): - """The the FromTimeString function.""" - # Test daylight savings. - expected_timestamp = timelib.Timestamp.CopyFromString( - '2013-10-01 12:00:00') - - # Check certain variance of this timestamp. - timestamp = timelib.Timestamp.FromTimeString( - '2013-10-01 14:00:00', timezone=pytz.timezone('Europe/Rome')) - self.assertEqual(timestamp, expected_timestamp) - - timestamp = timelib.Timestamp.FromTimeString( - '2013-10-01 12:00:00', timezone=pytz.timezone('UTC')) - self.assertEqual(timestamp, expected_timestamp) - - timestamp = timelib.Timestamp.FromTimeString( - '2013-10-01 05:00:00', timezone=pytz.timezone('PST8PDT')) - self.assertEqual(timestamp, expected_timestamp) - - # Now to test outside of the daylight savings. - expected_timestamp = timelib.Timestamp.CopyFromString( - '2014-02-01 12:00:00') - - timestamp = timelib.Timestamp.FromTimeString( - '2014-02-01 13:00:00', timezone=pytz.timezone('Europe/Rome')) - self.assertEqual(timestamp, expected_timestamp) - - timestamp = timelib.Timestamp.FromTimeString( - '2014-02-01 12:00:00', timezone=pytz.timezone('UTC')) - self.assertEqual(timestamp, expected_timestamp) - - timestamp = timelib.Timestamp.FromTimeString( - '2014-02-01 04:00:00', timezone=pytz.timezone('PST8PDT')) - self.assertEqual(timestamp, expected_timestamp) - - # Define two timestamps, one being GMT and the other UTC. - time_string_utc = 'Wed 05 May 2010 03:52:31 UTC' - time_string_gmt = 'Wed 05 May 2010 03:52:31 GMT' - - timestamp_utc = timelib.Timestamp.FromTimeString(time_string_utc) - timestamp_gmt = timelib.Timestamp.FromTimeString(time_string_gmt) - - # Test if these two are different, and if so, then we'll try again - # using the 'gmt_is_utc' flag, which then should result to the same - # results. - if timestamp_utc != timestamp_gmt: - self.assertEqual(timestamp_utc, timelib.Timestamp.FromTimeString( - time_string_gmt, gmt_as_timezone=False)) - - timestamp = timelib.Timestamp.FromTimeString( - '12-15-1984 05:13:00', timezone=pytz.timezone('EST5EDT')) - self.assertEqual(timestamp, 471953580000000) - - # Swap day and month. - timestamp = timelib.Timestamp.FromTimeString( - '12-10-1984 05:13:00', timezone=pytz.timezone('EST5EDT'), - dayfirst=True) - self.assertEqual(timestamp, 466420380000000) - - timestamp = timelib.Timestamp.FromTimeString('12-15-1984 10:13:00Z') - self.assertEqual(timestamp, 471953580000000) - - # Setting the timezone for string that already contains a timezone - # indicator should not affect the conversion. - timestamp = timelib.Timestamp.FromTimeString( - '12-15-1984 10:13:00Z', timezone=pytz.timezone('EST5EDT')) - self.assertEqual(timestamp, 471953580000000) - - timestamp = timelib.Timestamp.FromTimeString('15/12/1984 10:13:00Z') - self.assertEqual(timestamp, 471953580000000) - - timestamp = timelib.Timestamp.FromTimeString('15-12-84 10:13:00Z') - self.assertEqual(timestamp, 471953580000000) - - timestamp = timelib.Timestamp.FromTimeString( - '15-12-84 10:13:00-04', timezone=pytz.timezone('EST5EDT')) - self.assertEqual(timestamp, 471967980000000) - - with self.assertRaises(errors.TimestampError): - timestamp = timelib.Timestamp.FromTimeString( - 'thisisnotadatetime', timezone=pytz.timezone('EST5EDT')) - - timestamp = timelib.Timestamp.FromTimeString( - '12-15-1984 04:13:00', timezone=pytz.timezone('America/Chicago')) - self.assertEqual(timestamp, 471953580000000) - - timestamp = timelib.Timestamp.FromTimeString( - '07-14-1984 23:13:00', timezone=pytz.timezone('America/Chicago')) - self.assertEqual(timestamp, 458712780000000) - - timestamp = timelib.Timestamp.FromTimeString( - '12-15-1984 05:13:00', timezone=pytz.timezone('US/Pacific')) - self.assertEqual(timestamp, 471964380000000) - def testRoundTimestamp(self): """Test the RoundToSeconds function.""" # Should be rounded up.