Django 1.7+ and Galatea update

README.rst

@@ -5,8 +5,8 @@ django-eve-db
 :Author: Greg Taylor
 :License: BSD
 
-django-eve-db is a set of `Django`_ models that wrap CCP's EVE Online data 
-dump. This makes it extremely easy to query the dump and rapidly 
+django-eve-db is a set of `Django`_ 1.7+ models that wrap CCP's EVE Online data
+dump. This makes it extremely easy to query the dump and rapidly
 develop software around it. Little to no SQL is required for many usage cases.
 
 Source: https://github.com/gtaylor/django-eve-db
@@ -17,7 +17,7 @@ Source: https://github.com/gtaylor/django-eve-db
 Getting Started
 ---------------
 
-For details on how to get started using this software, see the 
+For details on how to get started using this software, see the
 `Getting Started`_ page.
 
 .. _Getting Started: https://github.com/gtaylor/django-eve-db/wiki/Getting-started

eve_db/__init__.py

@@ -1 +1 @@
-VERSION = "1.0.13-95173"
+VERSION = "1.1.0-114618"

eve_db/admin.py

@@ -5,7 +5,7 @@
 from eve_db.models import *
 
 class InvCategoryAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name', 'description', 'is_published')
+    list_display = ('id', 'name', 'is_published')
 admin.site.register(InvCategory, InvCategoryAdmin)
 
 class ChrBloodlineAdmin(admin.ModelAdmin):
@@ -34,11 +34,11 @@ class InvMetaTypeAdmin(admin.ModelAdmin):
 admin.site.register(InvMetaType, InvMetaTypeAdmin)
 
 class InvGroupAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name', 'category', 'description')
+    list_display = ('id', 'name', 'category')
 admin.site.register(InvGroup, InvGroupAdmin)
 
 class InvTypeAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name', 'group', 'market_group', 'description')
+    list_display = ('id', 'name', 'market_group', 'description')
 admin.site.register(InvType, InvTypeAdmin)
 
 class InvFlagAdmin(admin.ModelAdmin):
@@ -72,10 +72,6 @@ class InvContrabandTypeAdmin(admin.ModelAdmin):
                     'confiscate_min_sec', 'attack_min_sec', 'fine_by_value')
 admin.site.register(InvContrabandType, InvContrabandTypeAdmin)
 
-class InvBlueprintTypeAdmin(admin.ModelAdmin):
-    list_display = ('blueprint_type', 'product_type', 'tech_level')
-admin.site.register(InvBlueprintType, InvBlueprintTypeAdmin)
-
 class InvItemAdmin(admin.ModelAdmin):
     list_display = ('id', 'type', 'owner', 'location', 'flag', 'quantity')
 admin.site.register(InvItem, InvItemAdmin)
@@ -92,32 +88,28 @@ class RamActivityAdmin(admin.ModelAdmin):
     list_display = ('id', 'name', 'description', 'icon_filename', 'is_published')
 admin.site.register(RamActivity, RamActivityAdmin)
 
-class RamAssemblyLineAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name', 'station', 'owner', 'activity',
-                    'minimum_char_security', 'cost_per_hour', 'next_free_time',
-                    'restriction_mask')
-admin.site.register(RamAssemblyLine, RamAssemblyLineAdmin)
-
 class RamAssemblyLineStationsAdmin(admin.ModelAdmin):
     list_display = ('id', 'station', 'assembly_line_type', 'quantity',
                     'station_type', 'owner', 'solar_system', 'region')
 admin.site.register(RamAssemblyLineStations, RamAssemblyLineStationsAdmin)
 
 class RamAssemblyLineTypeAdmin(admin.ModelAdmin):
     list_display = ('id', 'name', 'description', 'base_time_multiplier',
-                    'base_material_multiplier', 'activity',
-                    'min_cost_per_hour')
+                    'base_cost_multiplier', 'base_material_multiplier',
+                    'activity', 'min_cost_per_hour')
 admin.site.register(RamAssemblyLineType, RamAssemblyLineTypeAdmin)
 
 class RamAssemblyLineTypeDetailPerCategoryAdmin(admin.ModelAdmin):
     list_display = ('id', 'assembly_line_type', 'category',
-                    'time_multiplier', 'material_multiplier')
+                    'time_multiplier', 'cost_multiplier',
+                    'material_multiplier')
 admin.site.register(RamAssemblyLineTypeDetailPerCategory,
                     RamAssemblyLineTypeDetailPerCategoryAdmin)
 
 class RamAssemblyLineTypeDetailPerGroupAdmin(admin.ModelAdmin):
     list_display = ('id', 'assembly_line_type', 'group',
-                    'time_multiplier', 'material_multiplier')
+                    'time_multiplier', 'cost_multiplier',
+                    'material_multiplier')
 admin.site.register(RamAssemblyLineTypeDetailPerGroup,
                     RamAssemblyLineTypeDetailPerGroupAdmin)
 

eve_db/ccp_importer/bulkops.py

@@ -35,12 +35,19 @@ def insert_many(objects, using="default"):
     import django.db.models
     from django.db import connections
     con = connections[using]
-    
+
     model = objects[0].__class__
     fields = [f for f in model._meta.fields if not isinstance(f, django.db.models.AutoField)]
     parameters = []
     for o in objects:
-        parameters.append(tuple(f.get_db_prep_save(f.pre_save(o, True), connection=con) for f in fields))
+        try:
+            parameters.append(tuple(f.get_db_prep_save(f.pre_save(o, True), connection=con) for f in fields))
+        except ValueError:
+            print("BROKEN CONVERSION!")
+            print(o)
+            print("------------------")
+            print(objects)
+            print("------------------")
     table = model._meta.db_table
     column_names = ",".join(con.ops.quote_name(f.column) for f in fields)
     placeholders = ",".join(("%s",) * len(fields))
@@ -66,7 +73,7 @@ def update_many(objects, fields=[], using="default"):
 
     if not fields:
         raise ValueError("No fields to update, field names are %s." % names)
-    
+
     fields_with_pk = fields + [meta.pk]
     parameters = []
     for o in objects:

eve_db/ccp_importer/importers/chr.py

@@ -45,8 +45,8 @@ def import_row(self, row):
         if row['corporationID']:
             corporation = CrpNPCCorporation.objects.get_or_create(id=row['corporationID'])[0]
         # Make sure to commit this or we'll get a transaction management error
-        if solar_system or corporation:
-            transaction.commit()
+        #if solar_system or corporation:
+        #    transaction.commit()
 
         new_instance = self.model(id=row['factionID'],
                                   name=row['factionName'],

eve_db/ccp_importer/importers/importer_classes.py

@@ -2,9 +2,7 @@
 This module holds all importer related classes.
 """
 from progressbar import ProgressBar, Percentage, Bar, ETA
-from django import db
-from django.db import transaction
-from django.conf import settings
+from django.db import IntegrityError, transaction
 from eve_db.ccp_importer.bulkops import insert_many, update_many
 
 class SQLImporter(object):
@@ -31,7 +29,7 @@ def __init__(self):
         self.progress_update_interval = 0
         self.pbar = None
 
-    @transaction.commit_manually
+    @transaction.atomic
     def prep_and_run_importer(self, conn):
         """
         Prepares the SQLite objects, progress bars, and other things and
@@ -42,8 +40,6 @@ def prep_and_run_importer(self, conn):
         self._setup_progressbar()
         self.insert_only = self.model.objects.all().count() == 0
 
-        transaction.commit()
-
         inserts_bucket = []
         updates_bucket = []
         inserts_counter = 0
@@ -53,39 +49,37 @@ def prep_and_run_importer(self, conn):
         query_string = 'SELECT * FROM %s' % self.table_name
 
         try:
-            for new_obj, insert in (self.import_row(row) for row in self.cursor.execute(query_string)):
-                # Now we have either a new model instance, an existing model
-                # instance with updated fields or None == skip
-                if new_obj:
-                    if insert:
-                        inserts_bucket.append(new_obj)
-                        inserts_counter += 1
-                        if inserts_counter % batch_size == 0:
-                            insert_many(inserts_bucket)
-                            inserts_bucket = []
-                    else:
-                        updates_bucket.append(new_obj)
-                        updates_counter += 1
-                        if updates_counter % batch_size == 0:
-                            update_many(updates_bucket)
-                            updates_bucket = []
-
-                if self.itercount % self.progress_update_interval == 0:
-                    self._progress_handler()
-
-                self.itercount += 1
-
-    #                if settings.DEBUG:
-    #                    db.reset_queries()
-            if len(inserts_bucket) > 0:
-                insert_many(inserts_bucket)
-            if len(updates_bucket) > 0:
-                update_many(updates_bucket)
-        except Exception:
-            transaction.rollback()
+            with transaction.atomic():
+                for new_obj, insert in (self.import_row(row) for row in self.cursor.execute(query_string)):
+                    # Now we have either a new model instance, an existing model
+                    # instance with updated fields or None == skip
+                    if new_obj:
+                        if insert:
+                            inserts_bucket.append(new_obj)
+                            inserts_counter += 1
+                            if inserts_counter % batch_size == 0:
+                                insert_many(inserts_bucket)
+                                inserts_bucket = []
+                        else:
+                            updates_bucket.append(new_obj)
+                            updates_counter += 1
+                            if updates_counter % batch_size == 0:
+                                update_many(updates_bucket)
+                                updates_bucket = []
+
+                    if self.itercount % self.progress_update_interval == 0:
+                        self._progress_handler()
+
+                    self.itercount += 1
+
+        #                if settings.DEBUG:
+        #                    db.reset_queries()
+                if len(inserts_bucket) > 0:
+                    insert_many(inserts_bucket)
+                if len(updates_bucket) > 0:
+                    update_many(updates_bucket)
+        except IntegrityError:
             raise
-        else:
-            transaction.commit()
         # Progressbar to 100%.
         self.pbar.finish()
         # Clean up the cursor, free the memory.

eve_db/ccp_importer/importers/inventory.py

@@ -16,7 +16,6 @@ class Importer_invCategories(SQLImporter):
     pks = (('id', 'categoryID'),)
     field_map = (('name', 'categoryName'),
                  ('icon_id', 'iconID'),
-                 ('description', 'description'),
                  ('is_published', 'published', parse_int_bool))
 
 
@@ -26,11 +25,8 @@ class Importer_invGroups(SQLImporter):
     pks = (('id', 'groupID'),)
     field_map = (('name', 'groupName'),
                  ('category_id', 'categoryID'),
-                 ('description', 'description'),
                  ('icon_id', 'iconID'),
                  ('use_base_price', 'useBasePrice', parse_int_bool),
-                 ('allow_manufacture', 'allowManufacture', parse_int_bool),
-                 ('allow_recycle', 'allowRecycler', parse_int_bool),
                  ('allow_anchoring', 'anchorable', parse_int_bool),
                  ('is_anchored', 'anchored', parse_int_bool),
                  ('is_fittable_non_singleton', 'fittableNonSingleton', parse_int_bool),
@@ -63,16 +59,14 @@ class Importer_invTypes(SQLImporter):
     pks = (('id', 'typeID'),)
     field_map = (('name', 'typeName'),
                  ('description', 'description', parse_char_notnull),
-                 ('group_id', 'groupID'),
                  ('mass', 'mass'),
                  ('volume', 'volume'),
                  ('capacity', 'capacity'),
                  ('portion_size', 'portionSize'),
                  ('base_price', 'basePrice'),
                  ('market_group_id', 'marketGroupID'),
                  ('is_published', 'published', parse_int_bool),
-                 ('race_id', 'raceID'),
-                 ('chance_of_duplicating', 'chanceOfDuplicating'))
+                 ('race_id', 'raceID'))
 
 
 class Importer_invTypeMaterials(SQLImporter):
@@ -169,24 +163,6 @@ class Importer_invFlags(SQLImporter):
                  ('order', 'orderID'))
 
 
-class Importer_invBlueprintTypes(SQLImporter):
-    DEPENDENCIES = ['invTypes', 'invBlueprintTypes']
-    model = InvBlueprintType
-    pks = (('blueprint_type', 'blueprintTypeID'),)
-    field_map = (('product_type_id', 'productTypeID'),
-                 ('production_time', 'productionTime'),
-                 ('parent_blueprint_type_id', 'parentBlueprintTypeID'),
-                 ('tech_level', 'techLevel'),
-                 ('research_productivity_time', 'researchProductivityTime'),
-                 ('research_material_time', 'researchMaterialTime'),
-                 ('research_copy_time', 'researchCopyTime'),
-                 ('research_tech_time', 'researchTechTime'),
-                 ('productivity_modifier', 'productivityModifier'),
-                 ('material_modifier', 'materialModifier'),
-                 ('waste_factor', 'wasteFactor'),
-                 ('max_production_limit', 'maxProductionLimit'))
-
-
 class Importer_invControlTowerResourcePurposes(SQLImporter):
     DEPENDENCIES = ['invControlTowerResourcePurposes']
     model = InvPOSResourcePurpose
@@ -247,4 +223,4 @@ class Importer_invUniqueNames(SQLImporter):
     model = InvUniqueName
     pks = (('id', 'itemID'),)
     field_map = (('name', 'itemName'),
-                 ('group_id', 'groupID'))
+                 ('group_id', 'groupID'))

eve_db/ccp_importer/importers/station.py

@@ -24,6 +24,7 @@ class Importer_ramAssemblyLineTypes(SQLImporter):
                  ('base_time_multiplier', 'baseTimeMultiplier'),
                  ('description', 'description'),
                  ('base_material_multiplier', 'baseMaterialMultiplier'),
+                 ('base_cost_multiplier', 'baseCostMultiplier'),
                  ('volume', 'volume'),
                  ('activity_id', 'activityID'),
                  ('min_cost_per_hour', 'minCostPerHour'))
@@ -35,48 +36,13 @@ class Importer_staOperationServices(SQLImporter):
     pks = (('operation', 'operationID'), ('service', 'serviceID'))
 
 
-class Importer_ramAssemblyLines(SQLImporter):
-    DEPENDENCIES = ['ramActivities', 'ramAssemblyLineTypes', 'staStations',
-                    'crpNPCCorporations']
-    model = RamAssemblyLine
-    pks = (('id', 'assemblyLineID'),)
-
-    def __init__(self, *args, **kwargs):
-        super(Importer_ramAssemblyLines, self).__init__(*args, **kwargs)
-        self.field_map = (('assembly_line_type_id', 'assemblyLineTypeID'),
-                 ('station_id', 'containerID'),
-                 ('owner_id', 'ownerID'),
-                 ('activity_id', 'activityID'),
-                 ('name', 'assemblyLineTypeID', self.get_assembly_line_type_name),
-                 ('ui_grouping_id', 'UIGroupingID'),
-                 ('cost_install', 'costInstall'),
-                 ('cost_per_hour', 'costPerHour'),
-                 ('discount_per_good_standing_point', 'discountPerGoodStandingPoint'),
-                 ('surcharge_per_bad_standing_point', 'surchargePerBadStandingPoint'),
-                 ('minimum_standing', 'minimumStanding'),
-                 ('minimum_char_security', 'minimumCharSecurity'),
-                 ('minimum_corp_security', 'minimumCorpSecurity'),
-                 ('maximum_char_security', 'maximumCharSecurity'),
-                 ('maximum_corp_security', 'maximumCorpSecurity'),
-                 ('next_free_time', 'nextFreeTime'),
-                 ('restriction_mask', 'restrictionMask'))
-
-        # Retrieve and store all assembly type names by ID
-        self.assembly_line_type_names = {}
-        for type_id, name in (keyvalue for keyvalue in RamAssemblyLineType.objects.all().values_list('id', 'name')):
-            self.assembly_line_type_names[type_id] = name
-
-    def get_assembly_line_type_name(self, type_id):
-#        return RamAssemblyLineType.objects.get(id=type_id).name
-        return self.assembly_line_type_names.get(type_id)
-
-
 class Importer_ramAssemblyLineTypeDetailPerCategory(SQLImporter):
     DEPENDENCIES = ['ramAssemblyLineTypes', 'invCategories']
     model = RamAssemblyLineTypeDetailPerCategory
     pks = (('assembly_line_type', 'assemblyLineTypeID'),
            ('category', 'categoryID'))
     field_map = (('time_multiplier', 'timeMultiplier'),
+                 ('cost_multiplier', 'costMultiplier'),
                  ('material_multiplier', 'materialMultiplier'))
 
 
@@ -86,6 +52,7 @@ class Importer_ramAssemblyLineTypeDetailPerGroup(SQLImporter):
     pks = (('assembly_line_type', 'assemblyLineTypeID'),
            ('group', 'groupID'))
     field_map = (('time_multiplier', 'timeMultiplier'),
+                 ('cost_multiplier', 'costMultiplier'),
                  ('material_multiplier', 'materialMultiplier'))
 
 
@@ -149,15 +116,6 @@ class Importer_ramAssemblyLineStations(SQLImporter):
                  ('quantity', 'quantity'))
 
 
-class Importer_ramTypeRequirements(SQLImporter):
-    DEPENDENCIES = ['invTypes', 'ramActivities']
-    model = RamTypeRequirement
-    pks = (('type', 'typeID'), ('activity_type', 'activityID'),
-           ('required_type', 'requiredTypeID'))
-    field_map = (('quantity', 'quantity'),
-                 ('damage_per_job', 'damagePerJob'),
-                 ('recycle', 'recycle', parse_int_bool))
-
 
 class Importer_staStations(SQLImporter):
     DEPENDENCIES = ['invTypes', 'staOperations', 'staStationTypes',