Refactor transactions db ( #667 #665 )

src/sakia/data/entities/transaction.py

@@ -52,12 +52,13 @@ def parse_transaction_doc(tx_doc, pubkey, block_number, mediantime, txid):
         return None
 
     transaction = Transaction(currency=tx_doc.currency,
+                              pubkey=pubkey,
                               sha_hash=tx_doc.sha_hash,
                               written_block=block_number,
                               blockstamp=tx_doc.blockstamp,
                               timestamp=mediantime,
-                              signature=tx_doc.signatures[0],
-                              issuer=tx_doc.issuers[0],
+                              signatures=tx_doc.signatures,
+                              issuers=tx_doc.issuers,
                               receivers=receivers,
                               amount=amount,
                               amount_base=amount_base,
@@ -93,13 +94,14 @@ class Transaction:
     REFUSED = 8
     DROPPED = 16
 
-    currency      = attr.ib(convert=str, cmp=False)
+    currency      = attr.ib(convert=str)
+    pubkey        = attr.ib(convert=str)
     sha_hash      = attr.ib(convert=str)
     written_block = attr.ib(convert=int, cmp=False)
     blockstamp    = attr.ib(convert=block_uid, cmp=False)
     timestamp     = attr.ib(convert=int, cmp=False)
-    signature     = attr.ib(convert=str, cmp=False)
-    issuer        = attr.ib(convert=str, cmp=False)
+    signatures    = attr.ib(convert=attrs_tuple_of_str, cmp=False)
+    issuers       = attr.ib(convert=attrs_tuple_of_str, cmp=False)
     receivers     = attr.ib(convert=attrs_tuple_of_str, cmp=False)
     amount        = attr.ib(convert=int, cmp=False)
     amount_base   = attr.ib(convert=int, cmp=False)

src/sakia/data/processors/transactions.py

@@ -82,8 +82,8 @@ class TransactionsProcessor:
         except sqlite3.IntegrityError:
             self._repo.update(tx)
 
-    def find_by_hash(self, sha_hash):
-        return self._repo.get_one(sha_hash=sha_hash)
+    def find_by_hash(self, pubkey, sha_hash):
+        return self._repo.get_one(pubkey=pubkey, sha_hash=sha_hash)
 
     def awaiting(self, currency):
         return self._repo.get_all(currency=currency, state=Transaction.AWAITING)
@@ -158,12 +158,6 @@ class TransactionsProcessor:
         :param List[str] connections_pubkeys: pubkeys of existing connections
         :return:
         """
-        sent = self._repo.get_all(currency=connection.currency, issuer=connection.pubkey)
-        for tx in sent:
-            for pubkey in connections_pubkeys:
-                if pubkey not in tx.receivers:
-                    self._repo.drop(tx)
-        received = self._repo.get_all(currency=connection.currency, receiver=connection.pubkey)
-        for tx in received:
-            if tx.issuer not in connections_pubkeys:
+        transactions = self._repo.get_all(currency=connection.currency, pubkey=connection.pubkey)
+        for tx in transactions:
             self._repo.drop(tx)

src/sakia/data/repositories/004_refactor_transactions.sql

+BEGIN TRANSACTION ;
+
+ALTER TABLE transactions RENAME TO TempOldTransactions;
+
+-- TRANSACTIONS TABLE
+CREATE TABLE IF NOT EXISTS transactions(
+                               currency VARCHAR(30),
+                               pubkey VARCHAR(50),
+                               sha_hash VARCHAR(50),
+                               written_on INT,
+                               blockstamp VARCHAR(100),
+                               ts INT,
+                               signatures VARCHAR(100),
+                               issuers TEXT,
+                               receivers TEXT,
+                               amount INT,
+                               amountbase INT,
+                               comment VARCHAR(255),
+                               txid INT,
+                               state INT,
+                               local BOOLEAN,
+                               raw TEXT,
+                               PRIMARY KEY (currency, pubkey, sha_hash)
+                               );
+
+DROP TABLE TempOldTransactions;
+
+COMMIT;
\ No newline at end of file

src/sakia/data/repositories/meta.py

@@ -7,7 +7,7 @@ from .connections import ConnectionsRepo
 from .identities import IdentitiesRepo
 from .blockchains import BlockchainsRepo
 from .certifications import CertificationsRepo
-from .transactions import TransactionsRepo
+from .transactions import TransactionsRepo, Transaction
 from .dividends import DividendsRepo
 from .nodes import NodesRepo
 from .sources import SourcesRepo
@@ -67,7 +67,8 @@ class SakiaDatabase:
             self.add_ud_rythm_parameters,
             self.add_contacts,
             self.add_sentry_property,
-            self.add_last_state_change_property
+            self.add_last_state_change_property,
+            self.refactor_transactions
         ]
 
     def upgrade_database(self, to=0):
@@ -135,6 +136,16 @@ class SakiaDatabase:
         with self.conn:
             self.conn.executescript(sql_file.read())
 
+    def refactor_transactions(self):
+        """
+        Init all the tables
+        :return:
+        """
+        self._logger.debug("Refactor transactions")
+        sql_file = open(os.path.join(os.path.dirname(__file__), '004_refactor_transactions.sql'), 'r')
+        with self.conn:
+            self.conn.executescript(sql_file.read())
+
     def version(self):
         with self.conn:
             c = self.conn.execute("SELECT * FROM meta WHERE id=1")

src/sakia/data/repositories/transactions.py

@@ -8,7 +8,7 @@ class TransactionsRepo:
     """The repository for Communities entities.
     """
     _conn = attr.ib()  # :type sqlite3.Connection
-    _primary_keys = (Transaction.sha_hash,)
+    _primary_keys = (Transaction.currency, Transaction.pubkey, Transaction.sha_hash,)
 
     def insert(self, transaction):
         """
@@ -17,7 +17,9 @@ class TransactionsRepo:
         """
         transaction_tuple = attr.astuple(transaction, tuple_factory=list)
 
+        transaction_tuple[6] = "\n".join([str(n) for n in transaction_tuple[6]])
         transaction_tuple[7] = "\n".join([str(n) for n in transaction_tuple[7]])
+        transaction_tuple[8] = "\n".join([str(n) for n in transaction_tuple[8]])
 
         values = ",".join(['?'] * len(transaction_tuple))
         self._conn.execute("INSERT INTO transactions VALUES ({0})".format(values), transaction_tuple)
@@ -29,18 +31,19 @@ class TransactionsRepo:
         """
         updated_fields = attr.astuple(transaction, filter=attr.filters.exclude(*TransactionsRepo._primary_keys),
                                       tuple_factory=list)
-        updated_fields[6] = "\n".join([str(n) for n in updated_fields[6]])
+        updated_fields[3] = "\n".join([str(n) for n in updated_fields[3]])
+        updated_fields[4] = "\n".join([str(n) for n in updated_fields[4]])
+        updated_fields[5] = "\n".join([str(n) for n in updated_fields[5]])
 
         where_fields = attr.astuple(transaction, filter=attr.filters.include(*TransactionsRepo._primary_keys),
                                     tuple_factory=list)
         self._conn.execute("""UPDATE transactions SET
-                           currency=?,
                            written_on=?,
                            blockstamp=?,
                            ts=?,
-                           signature=?,
-                           issuer = ?,
-                           receiver = ?,
+                           signatures=?,
+                           issuers = ?,
+                           receivers = ?,
                            amount = ?,
                            amountbase = ?,
                            comment = ?,
@@ -49,6 +52,8 @@ class TransactionsRepo:
                            local = ?,
                            raw = ?
                            WHERE
+                           currency=? AND
+                           pubkey=? AND
                            sha_hash=?""",
                            updated_fields + where_fields)
 
@@ -100,7 +105,7 @@ class TransactionsRepo:
         :rtype: List[sakia.data.entities.Transaction]
         """
         request = """SELECT * FROM transactions
-                  WHERE currency=? AND (issuer=? or receiver LIKE ?)
+                  WHERE currency=? AND pubkey=?
                   ORDER BY {sort_by} {sort_order}
                   LIMIT {limit} OFFSET {offset}""" \
                     .format(offset=offset,
@@ -108,7 +113,7 @@ class TransactionsRepo:
                             sort_by=sort_by,
                             sort_order=sort_order
                             )
-        c = self._conn.execute(request, (currency, pubkey, "%" + pubkey + "%"))
+        c = self._conn.execute(request, (currency, pubkey))
         datas = c.fetchall()
         if datas:
             return [Transaction(*data) for data in datas]
@@ -122,4 +127,6 @@ class TransactionsRepo:
         where_fields = attr.astuple(transaction, filter=attr.filters.include(*TransactionsRepo._primary_keys))
         self._conn.execute("""DELETE FROM transactions
                               WHERE
+                              currency=? AND
+                              pubkey=? AND
                               sha_hash=?""", where_fields)

src/sakia/gui/navigation/txhistory/table_model.py

@@ -256,18 +256,20 @@ class HistoryTableModel(QAbstractTableModel):
 
         amount = transfer.amount * 10**transfer.amount_base
 
-        identity = self.identities_service.get_identity(transfer.issuer)
+        senders = []
+        for issuer in transfer.issuers:
+            identity = self.identities_service.get_identity(issuer)
             if identity:
-            sender = identity.uid
+                senders.append(identity.uid)
             else:
-            sender = transfer.issuer
+                senders.append(issuer)
 
         date_ts = transfer.timestamp
         txid = transfer.txid
 
-        return (date_ts, sender, amount,
+        return (date_ts, "\n".join(senders), amount,
                 transfer.comment, transfer.state, txid,
-                transfer.issuer, block_number, transfer.sha_hash, transfer)
+                transfer.issuers, block_number, transfer.sha_hash, transfer)
 
     def data_sent(self, transfer):
         """
@@ -289,7 +291,7 @@ class HistoryTableModel(QAbstractTableModel):
         date_ts = transfer.timestamp
         txid = transfer.txid
         return (date_ts, "\n".join(receivers), amount, transfer.comment, transfer.state, txid,
-                "\n".join(transfer.receivers), block_number, transfer.sha_hash, transfer)
+                transfer.receivers, block_number, transfer.sha_hash, transfer)
 
     def data_dividend(self, dividend):
         """
@@ -316,7 +318,7 @@ class HistoryTableModel(QAbstractTableModel):
         transfers = self.transfers()
         for transfer in transfers:
             if transfer.state != Transaction.DROPPED:
-                if transfer.issuer == self.connection.pubkey:
+                if self.connection.pubkey in transfer.issuers:
                     self.transfers_data.append(self.data_sent(transfer))
                 if self.connection.pubkey in transfer.receivers:
                     self.transfers_data.append(self.data_received(transfer))

src/sakia/services/documents.py

@@ -395,13 +395,14 @@ class DocumentsService:
         self.commit_outputs_to_self(currency, key.pubkey, txdoc)
         time = self._blockchain_processor.time(currency)
         tx = Transaction(currency=currency,
+                         pubkey=key.pubkey,
                          sha_hash=txdoc.sha_hash,
                          written_block=0,
                          blockstamp=blockstamp,
                          timestamp=time,
-                         signature=txdoc.signatures[0],
-                         issuer=key.pubkey,
-                         receivers=receiver,
+                         signatures=txdoc.signatures,
+                         issuers=[key.pubkey],
+                         receivers=[receiver],
                          amount=amount,
                          amount_base=amount_base,
                          comment=txdoc.comment,

src/sakia/services/sources.py

@@ -87,13 +87,14 @@ class SourcesServices(QObject):
                 next_txid = self._transactions_processor.next_txid(self.currency, block_number)
                 sha_identifier = hashlib.sha256("Destruction{0}{1}{2}".format(block_number, pubkey, amount).encode("ascii")).hexdigest().upper()
                 destruction = Transaction(currency=self.currency,
+                                          pubkey=pubkey,
                                           sha_hash=sha_identifier,
                                           written_block=block_number,
                                           blockstamp=BlockUID.empty(),
                                           timestamp=timestamp,
-                                          signature="",
-                                          issuer=pubkey,
-                                          receivers="",
+                                          signatures=[],
+                                          issuers=[pubkey],
+                                          receivers=[],
                                           amount=amount,
                                           amount_base=0,
                                           comment="Too low balance",

src/sakia/services/transactions.py

@@ -47,12 +47,12 @@ class TransactionsService(QObject):
             if self._transactions_processor.run_state_transitions(tx, block_doc):
                 transfers_changed.append(tx)
                 self._logger.debug("New transaction validated : {0}".format(tx.sha_hash))
-
-        new_transactions = [t for t in block_doc.transactions
-                            if not self._transactions_processor.find_by_hash(t.sha_hash)
-                            and SimpleTransaction.is_simple(t)]
         connections = self._connections_processor.connections_to(self.currency)
         for conn in connections:
+            new_transactions = [t for t in block_doc.transactions
+                                if not self._transactions_processor.find_by_hash(conn.pubkey, t.sha_hash)
+                                and SimpleTransaction.is_simple(t)]
+
             new_transfers[conn] = []
             for (i, tx_doc) in enumerate(new_transactions):
                 tx = parse_transaction_doc(tx_doc, conn.pubkey, block_doc.blockUID.number,  block_doc.mediantime, txid+i)

tests/unit/data/test_transactions_repo.py

@@ -5,6 +5,7 @@ from sakia.data.entities import Transaction
 def test_add_get_drop_transaction(meta_repo):
     transactions_repo = TransactionsRepo(meta_repo.conn)
     transactions_repo.insert(Transaction("testcurrency",
+                                         "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
                                          "FCAD5A388AC8A811B45A9334A375585E77071AA9F6E5B6896582961A6C66F365",
                                          20,
                                          "15-76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67",
@@ -24,10 +25,10 @@ def test_add_get_drop_transaction(meta_repo):
     assert transaction.blockstamp.number == 15
     assert transaction.blockstamp.sha_hash == "76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67"
     assert transaction.timestamp == 1473108382
-    assert transaction.signature == "H41/8OGV2W4CLKbE35kk5t1HJQsb3jEM0/QGLUf80CwJvGZf3HvVCcNtHPUFoUBKEDQO9mPK3KJkqOoxHpqHCw=="
+    assert transaction.signatures[0] == "H41/8OGV2W4CLKbE35kk5t1HJQsb3jEM0/QGLUf80CwJvGZf3HvVCcNtHPUFoUBKEDQO9mPK3KJkqOoxHpqHCw=="
     assert transaction.amount == 1565
     assert transaction.amount_base == 1
-    assert transaction.issuer == "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ"
+    assert transaction.issuers[0] == "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ"
     assert transaction.receivers[0] == "FADxcH5LmXGmGFgdixSes6nWnC4Vb4pRUBYT81zQRhjn"
     assert transaction.comment == ""
     assert transaction.txid == 0
@@ -39,6 +40,7 @@ def test_add_get_drop_transaction(meta_repo):
 def test_add_get_multiple_transaction(meta_repo):
     transactions_repo = TransactionsRepo(meta_repo.conn)
     transactions_repo.insert(Transaction("testcurrency",
+                                         "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
                                          "A0AC57E2E4B24D66F2D25E66D8501D8E881D9E6453D1789ED753D7D426537ED5",
                                          12,
                                          "543-76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67",
@@ -52,6 +54,7 @@ def test_add_get_multiple_transaction(meta_repo):
                                          2,
                                          Transaction.TO_SEND))
     transactions_repo.insert(Transaction("testcurrency",
+                                         "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
                                          "FCAD5A388AC8A811B45A9334A375585E77071AA9F6E5B6896582961A6C66F365",
                                          20,
                                          "15-76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67",
@@ -67,24 +70,25 @@ def test_add_get_multiple_transaction(meta_repo):
     transactions = transactions_repo.get_all(currency="testcurrency")
     assert "testcurrency" in [t.currency for t in transactions]
     assert "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ" in [t.receivers[0] for t in transactions]
-    assert "FADxcH5LmXGmGFgdixSes6nWnC4Vb4pRUBYT81zQRhjn" in [t.issuer for t in transactions]
+    assert "FADxcH5LmXGmGFgdixSes6nWnC4Vb4pRUBYT81zQRhjn" in [t.issuers[0] for t in transactions]
 
 
 def test_add_update_transaction(meta_repo):
     transactions_repo = TransactionsRepo(meta_repo.conn)
-    transaction = Transaction("testcurrency",
-                              "FCAD5A388AC8A811B45A9334A375585E77071AA9F6E5B6896582961A6C66F365",
-                              20,
-                              "15-76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67",
-                              1473108382,
-                              "H41/8OGV2W4CLKbE35kk5t1HJQsb3jEM0/QGLUf80CwJvGZf3HvVCcNtHPUFoUBKEDQO9mPK3KJkqOoxHpqHCw==",
-                              "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
-                              "FADxcH5LmXGmGFgdixSes6nWnC4Vb4pRUBYT81zQRhjn",
-                              1565,
-                              1,
-                              "",
-                              0,
-                              Transaction.TO_SEND)
+    transaction = Transaction(currency      = "testcurrency",
+                              pubkey        = "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
+                              sha_hash      = "FCAD5A388AC8A811B45A9334A375585E77071AA9F6E5B6896582961A6C66F365",
+                              written_block = 20,
+                              blockstamp    = "15-76543400E78B56CC21FB1DDC6CBAB24E0FACC9A798F5ED8736EA007F38617D67",
+                              timestamp     = 1473108382,
+                              signatures    = "H41/8OGV2W4CLKbE35kk5t1HJQsb3jEM0/QGLUf80CwJvGZf3HvVCcNtHPUFoUBKEDQO9mPK3KJkqOoxHpqHCw==",
+                              issuers       = "7Aqw6Efa9EzE7gtsc8SveLLrM7gm6NEGoywSv4FJx6pZ",
+                              receivers     = "FADxcH5LmXGmGFgdixSes6nWnC4Vb4pRUBYT81zQRhjn",
+                              amount        = 1565,
+                              amount_base   = 1,
+                              comment       = "",
+                              txid          = 0,
+                              state         = Transaction.TO_SEND)
     transactions_repo.insert(transaction)
     transaction.written_on = None
     transactions_repo.update(transaction)