Scrapy - processing several types of elements - multiple and related Django models and storing them in a database in pipelines

Question

I have the following Django models. I'm not sure the best way to save these interconnected objects when scanning to a spider in a database in Django using scrapy protocols. It looks like the piping for scripting was built to handle only one “view” of an item

models.py

class Parent(models.Model):
    field1 = CharField()


class ParentX(models.Model):
    field2 = CharField()
    parent = models.OneToOneField(Parent, related_name = 'extra_properties')


class Child(models.Model):
    field3 = CharField()
    parent = models.ForeignKey(Parent, related_name='childs')

items.py

# uses DjangoItem https://github.com/scrapy-plugins/scrapy-djangoitem

class ParentItem(DjangoItem):
    django_model = Parent

class ParentXItem(DjangoItem):
    django_model = ParentX

class ChildItem(DjangoItem):
    django_model = Child

spiders.py

class MySpider(scrapy.Spider):
    name = "myspider"
    allowed_domains = ["abc.com"]
    start_urls = [
        "http://www.example.com",       # this page has ids of several Parent objects whose full details are in their individual pages

    ]

    def parse(self, response):
        parent_object_ids = [] #list from scraping the ids of the parent objects

        for parent_id in parent_object_ids:
            url = "http://www.example.com/%s" % parent_id
            yield scrapy.Request(url, callback=self.parse_detail)

    def parse_detail(self, response):
        p = ParentItem()
        px = ParentXItem()
        c = ChildItem()



        # populate p, px and c1, c2 with various data from the response.body

        yield p
        yield px
        yield c1
        yield c2 ... etc c3, c4

pipelines.py - not sure what to do here

class ScrapytestPipeline(object):
    def process_item(self, item, spider):


        # This is where typically storage to database happens
        # Now, I dont know whether the item is a ParentItem or ParentXItem or ChildItem

        # Ideally, I want to first create the Parent obj and then ParentX obj (and point p.extra_properties = px), and then child objects 
        # c1.parent = p, c2.parent = p

        # But I am not sure how to have pipeline do this in a sequential way from any order of items received

Answer 1

, , , - , .

, db.

spiders.py, u " p, px c1, c2 response.body", "" , .

u , :

class ItemPersistencePipeline(object):
    def process_item(self, item, spider):
        try:
             item_model = item_to_model(item)
        except TypeError:
            return item   
        model, created = get_or_create(item_model)
        try:
            update_model(model, item_model)
        except Exception,e:
            return e
        return item

, :

def item_to_model(item):
    model_class = getattr(item, 'django_model')
    if not model_class:
        raise TypeError("Item is not a `DjangoItem` or is misconfigured")   
    return item.instance   

def get_or_create(model):
    model_class = type(model)
    created = False
    try:
        #We have no unique identifier at the moment
        #use the model.primary for now
        obj = model_class.objects.get(primary=model.primary)
    except model_class.DoesNotExist:
        created = True
        obj = model  # DjangoItem created a model for us.

    return (obj, created)

from django.forms.models import model_to_dict

def update_model(destination, source, commit=True):
    pk = destination.pk

    source_dict = model_to_dict(source)
    for (key, value) in source_dict.items():
        setattr(destination, key, value)

    setattr(destination, 'pk', pk)

    if commit:
        destination.save()

    return destination

from: DjangoItem Scrapy

"primary" django , scraped

models.py

class Parent(models.Model):
    field1 = CharField()   
    #primary_key=True
    primary = models.CharField(max_length=80)
class ParentX(models.Model):
    field2 = CharField()
    parent = models.OneToOneField(Parent, related_name = 'extra_properties')
    primary = models.CharField(max_length=80) 
class Child(models.Model):
    field3 = CharField()
    parent = models.ForeignKey(Parent, related_name='childs')
    primary = models.CharField(max_length=80)

Answer 2

eLRuLL, isinstance , .

, , - , scrapy parent, parentX child.

, .

, .