Replaced timelib FromTimeString with dfDateTime #910

config/dpkg/control

@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python3-plaso
 Architecture: all
-Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
+Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200613), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

config/travis/install.sh

@@ -5,7 +5,7 @@
 # This file is generated by l2tdevtools update-dependencies.py any dependency
 # related changes should be made in dependencies.ini.
 
-DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libluksde-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-cffi python3-chardet python3-cryptography python3-dateutil python3-defusedxml python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-elasticsearch python3-future python3-idna python3-lz4 python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-redis python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq";
+DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libluksde-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-cffi-backend python3-chardet python3-cryptography python3-dateutil python3-defusedxml python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-elasticsearch python3-future python3-idna python3-lz4 python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-redis python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq";
 
 DPKG_PYTHON3_TEST_DEPENDENCIES="python3-coverage python3-distutils python3-fakeredis python3-mock python3-pbr python3-setuptools python3-sortedcontainers";
 

dependencies.ini

@@ -55,7 +55,7 @@ version_property: __version__
 
 [dfdatetime]
 dpkg_name: python3-dfdatetime
-minimum_version: 20200501
+minimum_version: 20200613
 rpm_name: python3-dfdatetime
 version_property: __version__
 

plaso/dependencies.py

@@ -27,7 +27,7 @@
     'cryptography': ('__version__', '2.0.2', None, True),
     'dateutil': ('__version__', '1.5', None, True),
     'defusedxml': ('__version__', '0.5.0', None, True),
-    'dfdatetime': ('__version__', '20200501', None, True),
+    'dfdatetime': ('__version__', '20200613', None, True),
     'dfvfs': ('__version__', '20200604', None, True),
     'dfwinreg': ('__version__', '20180712', None, True),
     'dtfabric': ('__version__', '20181128', None, True),

plaso/lib/timelib.py

@@ -11,15 +11,12 @@
 
 from __future__ import unicode_literals
 
-import calendar
 import datetime
 import logging
 
-import dateutil.parser
 import pytz
 
 from plaso.lib import definitions
-from plaso.lib import errors
 
 # pylint: disable=missing-type-doc,missing-return-type-doc
 
@@ -91,55 +88,6 @@ def CopyToIsoFormat(cls, timestamp, timezone=pytz.UTC, raise_error=False):
 
     return datetime_object.isoformat()
 
-  @classmethod
-  def FromTimeString(
-      cls, time_string, dayfirst=False, gmt_as_timezone=True,
-      timezone=pytz.UTC):
-    """Converts a string containing a date and time value into a timestamp.
-
-    Args:
-      time_string: String that contains a date and time value.
-      dayfirst: An optional boolean argument. If set to true then the
-                parser will change the precedence in which it parses timestamps
-                from MM-DD-YYYY to DD-MM-YYYY (and YYYY-MM-DD will be
-                YYYY-DD-MM, etc).
-      gmt_as_timezone: Sometimes the dateutil parser will interpret GMT and UTC
-                       the same way, that is not make a distinction. By default
-                       this is set to true, that is GMT can be interpreted
-                       differently than UTC. If that is not the expected result
-                       this attribute can be set to false.
-      timezone: Optional timezone object (instance of pytz.timezone) that
-                the data and time value in the string represents. This value
-                is used when the timezone cannot be determined from the string.
-
-    Returns:
-      The timestamp which is an integer containing the number of microseconds
-      since January 1, 1970, 00:00:00 UTC or 0 on error.
-
-    Raises:
-      TimestampError: if the time string could not be parsed.
-    """
-    if not gmt_as_timezone and time_string.endswith(' GMT'):
-      time_string = '{0:s}UTC'.format(time_string[:-3])
-
-    try:
-      # TODO: deprecate the use of dateutil parser.
-      datetime_object = dateutil.parser.parse(time_string, dayfirst=dayfirst)
-
-    except (TypeError, ValueError) as exception:
-      raise errors.TimestampError((
-          'Unable to convert time string: {0:s} in to a datetime object '
-          'with error: {1!s}').format(time_string, exception))
-
-    if datetime_object.tzinfo:
-      datetime_object = datetime_object.astimezone(pytz.UTC)
-    else:
-      datetime_object = timezone.localize(datetime_object)
-
-    posix_time = int(calendar.timegm(datetime_object.utctimetuple()))
-    timestamp = posix_time * definitions.MICROSECONDS_PER_SECOND
-    return timestamp + datetime_object.microsecond
-
   @classmethod
   def LocaltimeToUTC(cls, timestamp, timezone, is_dst=False):
     """Converts the timestamp in localtime of the timezone to UTC.

plaso/parsers/java_idx.py

@@ -10,12 +10,13 @@
 #    be present. 6.02 files will currently return 'Unknown'.
 
 from dfdatetime import java_time as dfdatetime_java_time
+from dfdatetime import semantic_time as dfdatetime_semantic_time
+from dfdatetime import time_elements as dfdatetime_time_elements
 
 from plaso.containers import events
 from plaso.containers import time_events
 from plaso.lib import errors
 from plaso.lib import definitions
-from plaso.lib import timelib
 from plaso.parsers import dtfabric_parser
 from plaso.parsers import manager
 
@@ -158,22 +159,23 @@ def ParseFileObject(self, parser_mediator, file_object):
       parser_mediator.ProduceEventWithEventData(event, event_data)
 
     if date_http_header:
-      # A HTTP header date and string "should" be in UTC or have an associated
-      # time zone information in the string itself. If that is not the case
-      # then there is no reliable method for plaso to determine the proper
-      # time zone, so the assumption is that it is UTC.
+      # A HTTP header date and time should be formatted according to RFC 1123.
+      # The date "should" be in UTC or have associated time zone information
+      # in the string itself. If that is not the case then there is no reliable
+      # method for Plaso to determine the proper time zone, so the assumption
+      # is that the date and time is in UTC.
       try:
-        download_date = timelib.Timestamp.FromTimeString(
-            date_http_header.value, gmt_as_timezone=False)
-      except errors.TimestampError:
-        parser_mediator.ProduceExtractionWarning(
-            'Unable to parse date HTTP header value: {0:s}'.format(
-                date_http_header.value))
+        date_time = dfdatetime_time_elements.TimeElements()
+        date_time.CopyFromStringRFC1123(date_http_header.value)
+      except ValueError as exception:
+        parser_mediator.ProduceExtractionWarning((
+            'Unable to parse date HTTP header string: {0:s} with error: '
+            '{1!s}').format(date_http_header.value, exception))
+        date_time = dfdatetime_semantic_time.InvalidTime()
 
-      if download_date:
-        event = time_events.TimestampEvent(
-            download_date, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED)
-        parser_mediator.ProduceEventWithEventData(event, event_data)
+      event = time_events.DateTimeValuesEvent(
+          date_time, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED)
+      parser_mediator.ProduceEventWithEventData(event, event_data)
 
 
 manager.ParsersManager.RegisterParser(JavaIDXParser)

requirements.txt

@@ -9,7 +9,7 @@ cffi >= 1.9.1
 chardet >= 2.0.1
 cryptography >= 2.0.2
 defusedxml >= 0.5.0
-dfdatetime >= 20180704
+dfdatetime >= 20200613
 dfvfs >= 20200604
 dfwinreg >= 20180712
 dtfabric >= 20181128

setup.cfg

@@ -51,7 +51,7 @@ requires = libbde-python3 >= 20140531
            python3-cryptography >= 2.0.2
            python3-dateutil >= 1.5
            python3-defusedxml >= 0.5.0
-           python3-dfdatetime >= 20180704
+           python3-dfdatetime >= 20200613
            python3-dfvfs >= 20200604
            python3-dfwinreg >= 20180712
            python3-dtfabric >= 20181128

tests/lib/timelib.py

@@ -6,7 +6,6 @@
 
 import unittest
 
-from plaso.lib import errors
 from plaso.lib import timelib
 
 import pytz  # pylint: disable=wrong-import-order
@@ -105,100 +104,6 @@ def testLocaltimeToUTC(self):
         '2013-01-01 05:00:00')
     self.assertEqual(timestamp, expected_timestamp)
 
-  def testTimestampFromTimeString(self):
-    """The the FromTimeString function."""
-    # Test daylight savings.
-    expected_timestamp = shared_test_lib.CopyTimestampFromSring(
-        '2013-10-01 12:00:00')
-
-    # Check certain variance of this timestamp.
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2013-10-01 14:00:00', timezone=pytz.timezone('Europe/Rome'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2013-10-01 12:00:00', timezone=pytz.timezone('UTC'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2013-10-01 05:00:00', timezone=pytz.timezone('PST8PDT'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    # Now to test outside of the daylight savings.
-    expected_timestamp = shared_test_lib.CopyTimestampFromSring(
-        '2014-02-01 12:00:00')
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2014-02-01 13:00:00', timezone=pytz.timezone('Europe/Rome'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2014-02-01 12:00:00', timezone=pytz.timezone('UTC'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '2014-02-01 04:00:00', timezone=pytz.timezone('PST8PDT'))
-    self.assertEqual(timestamp, expected_timestamp)
-
-    # Define two timestamps, one being GMT and the other UTC.
-    time_string_utc = 'Wed 05 May 2010 03:52:31 UTC'
-    time_string_gmt = 'Wed 05 May 2010 03:52:31 GMT'
-
-    timestamp_utc = timelib.Timestamp.FromTimeString(time_string_utc)
-    timestamp_gmt = timelib.Timestamp.FromTimeString(time_string_gmt)
-
-    # Test if these two are different, and if so, then we'll try again
-    # using the 'gmt_is_utc' flag, which then should result to the same
-    # results.
-    if timestamp_utc != timestamp_gmt:
-      self.assertEqual(timestamp_utc, timelib.Timestamp.FromTimeString(
-          time_string_gmt, gmt_as_timezone=False))
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '12-15-1984 05:13:00', timezone=pytz.timezone('EST5EDT'))
-    self.assertEqual(timestamp, 471953580000000)
-
-    # Swap day and month.
-    timestamp = timelib.Timestamp.FromTimeString(
-        '12-10-1984 05:13:00', timezone=pytz.timezone('EST5EDT'),
-        dayfirst=True)
-    self.assertEqual(timestamp, 466420380000000)
-
-    timestamp = timelib.Timestamp.FromTimeString('12-15-1984 10:13:00Z')
-    self.assertEqual(timestamp, 471953580000000)
-
-    # Setting the timezone for string that already contains a timezone
-    # indicator should not affect the conversion.
-    timestamp = timelib.Timestamp.FromTimeString(
-        '12-15-1984 10:13:00Z', timezone=pytz.timezone('EST5EDT'))
-    self.assertEqual(timestamp, 471953580000000)
-
-    timestamp = timelib.Timestamp.FromTimeString('15/12/1984 10:13:00Z')
-    self.assertEqual(timestamp, 471953580000000)
-
-    timestamp = timelib.Timestamp.FromTimeString('15-12-84 10:13:00Z')
-    self.assertEqual(timestamp, 471953580000000)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '15-12-84 10:13:00-04', timezone=pytz.timezone('EST5EDT'))
-    self.assertEqual(timestamp, 471967980000000)
-
-    with self.assertRaises(errors.TimestampError):
-      timestamp = timelib.Timestamp.FromTimeString(
-          'thisisnotadatetime', timezone=pytz.timezone('EST5EDT'))
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '12-15-1984 04:13:00', timezone=pytz.timezone('America/Chicago'))
-    self.assertEqual(timestamp, 471953580000000)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '07-14-1984 23:13:00', timezone=pytz.timezone('America/Chicago'))
-    self.assertEqual(timestamp, 458712780000000)
-
-    timestamp = timelib.Timestamp.FromTimeString(
-        '12-15-1984 05:13:00', timezone=pytz.timezone('US/Pacific'))
-    self.assertEqual(timestamp, 471964380000000)
-
 
 if __name__ == '__main__':
   unittest.main()