Shapeshift

shapeshift

by Alex Azuero

JavaScript

import urllib.request
import time
import json
import csv
import os
import sys
import logging.handlers

transactions_temp = []
TOTAL_WRITTEN: int = 0
IN_MEMORY_MAX: int = 80     # Arbitrary number larger than the amount of results returned by the API


# Fast de-duplication
def f7(seq, key):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x[key] in seen or seen_add(x[key]))]


class RotatingCSVFile(object):
    def __init__(self, directory='', filename='scrape', ext='.csv', max_files=sys.maxsize,
                 max_file_size=1024 * 1024 * 100, dict_keys=None):
        self.ii = 1
        self.directory, self.filename = directory, filename
        self.max_file_size, self.max_files = max_file_size, max_files
        self.finished, self.fh = False, None
        self.ext = ext

        if dict_keys is None:
            dict_keys = []
        self.keys = dict_keys

        self.dict_writer = None
        self.open()

    def rotate(self):
        if os.stat(self.filename_template).st_size > self.max_file_size:
            self.close()
            self.ii += 1
            if self.ii <= self.max_files:
                self.open()
            else:
                # self.cleanup()
                self.close()
                self.finished = True

    def open(self):
        self.fh = open(self.filename_template, 'w')
        self.dict_writer = csv.DictWriter(self.fh, fieldnames=self.keys, lineterminator='\n')
        self.dict_writer.writeheader()
        self.fh.flush()

    def write(self, out):
        self.dict_writer.writerows(out)
        self.fh.flush()
        self.rotate()

    def close(self):
        self.fh.close()

    def cleanup(self, out):
        self.ii -= 1
        self.fh = open(self.filename_template, 'a')
        self.dict_writer = csv.DictWriter(self.fh, fieldnames=self.keys, lineterminator='\n')
        self.dict_writer.writerows(out)
        self.close()
        self.finished = True


    @property
    def...