Added audio length (in seconds) to database. Added relative confidence value. Created branch to avoid pull request #148 issues (more commits)

dejavu/__init__.py

@@ -15,6 +15,8 @@ class Dejavu(object):
     MATCH_TIME = 'match_time'
     OFFSET = 'offset'
     OFFSET_SECS = 'offset_seconds'
+    AUDIO_LENGTH = 'audio_length'
+    RELATIVE_CONFIDENCE = 'relative_confidence'
 
     def __init__(self, config):
         super(Dejavu, self).__init__()
@@ -74,7 +76,7 @@ def fingerprint_directory(self, path, extensions, nprocesses=None):
         # Loop till we have all of them
         while True:
             try:
-                song_name, hashes, file_hash = iterator.next()
+                song_name, hashes, file_hash, audio_length = iterator.next()
             except multiprocessing.TimeoutError:
                 continue
             except StopIteration:
@@ -84,12 +86,14 @@ def fingerprint_directory(self, path, extensions, nprocesses=None):
                 # Print traceback because we can't reraise it here
                 traceback.print_exc(file=sys.stdout)
             else:
-                sid = self.db.insert_song(song_name, file_hash)
+                print("Inserting " + song_name + " in database")
+                sid = self.db.insert_song(song_name, file_hash, audio_length)
 
                 self.db.insert_hashes(sid, hashes)
                 self.db.set_song_fingerprinted(sid)
                 self.get_fingerprinted_songs()
 
+                print(song_name + " inserted in database")
         pool.close()
         pool.join()
 
@@ -101,22 +105,29 @@ def fingerprint_file(self, filepath, song_name=None):
         if song_hash in self.songhashes_set:
             print "%s already fingerprinted, continuing..." % song_name
         else:
-            song_name, hashes, file_hash = _fingerprint_worker(
+            song_name, hashes, file_hash, audio_length = _fingerprint_worker(
                 filepath,
                 self.limit,
                 song_name=song_name
             )
-            sid = self.db.insert_song(song_name, file_hash)
+            print("Inserting " + song_name + " in database")
+            sid = self.db.insert_song(song_name, file_hash, audio_length)
 
             self.db.insert_hashes(sid, hashes)
             self.db.set_song_fingerprinted(sid)
             self.get_fingerprinted_songs()
+            print(song_name + " inserted in database")
 
     def find_matches(self, samples, Fs=fingerprint.DEFAULT_FS):
         hashes = fingerprint.fingerprint(samples, Fs=Fs)
-        return self.db.return_matches(hashes)
-
-    def align_matches(self, matches):
+        mapper = {}
+        total_hashes = 0
+        for hash, offset in hashes:
+            mapper[hash.upper()[:fingerprint.FINGERPRINT_REDUCTION]] = offset
+            total_hashes += 1
+        return (self.db.return_matches(mapper), total_hashes)
+
+    def align_matches(self, matches, total_hashes):
         """
             Finds hash matches that align in time with other matches and finds
             consensus about which hashes are "true" signal from the audio.
@@ -157,6 +168,8 @@ def align_matches(self, matches):
             Dejavu.SONG_ID : song_id,
             Dejavu.SONG_NAME : songname.encode("utf8"),
             Dejavu.CONFIDENCE : largest_count,
+            Dejavu.AUDIO_LENGTH : song.get(Database.AUDIO_LENGTH, None),
+            Dejavu.RELATIVE_CONFIDENCE : (largest_count*100)/float(total_hashes),
             Dejavu.OFFSET : int(largest),
             Dejavu.OFFSET_SECS : nseconds,
             Database.FIELD_FILE_SHA1 : song.get(Database.FIELD_FILE_SHA1, None).encode("utf8"),}
@@ -177,7 +190,7 @@ def _fingerprint_worker(filename, limit=None, song_name=None):
 
     songname, extension = os.path.splitext(os.path.basename(filename))
     song_name = song_name or songname
-    channels, Fs, file_hash = decoder.read(filename, limit)
+    channels, Fs, file_hash, audio_length = decoder.read(filename, limit)
     result = set()
     channel_amount = len(channels)
 
@@ -191,7 +204,7 @@ def _fingerprint_worker(filename, limit=None, song_name=None):
                                                  filename))
         result |= set(hashes)
 
-    return song_name, result, file_hash
+    return song_name, result, file_hash, audio_length
 
 
 def chunkify(lst, n):

dejavu/database.py

@@ -10,6 +10,7 @@ class Database(object):
     FIELD_SONGNAME = 'song_name'
     FIELD_OFFSET = 'offset'
     FIELD_HASH = 'hash'
+    AUDIO_LENGTH = 'audio_length'
 
     # Name of your Database subclass, this is used in configuration
     # to refer to your class

dejavu/database_sql.py

@@ -1,12 +1,15 @@
 from __future__ import absolute_import
 from itertools import izip_longest
 import Queue
+import math
 
 import MySQLdb as mysql
 from MySQLdb.cursors import DictCursor
 
 from dejavu.database import Database
+from dejavu.fingerprint import FINGERPRINT_REDUCTION
 
+from itertools import chain
 
 class SQLDatabase(Database):
     """
@@ -56,14 +59,14 @@ class SQLDatabase(Database):
     # creates
     CREATE_FINGERPRINTS_TABLE = """
         CREATE TABLE IF NOT EXISTS `%s` (
-             `%s` binary(10) not null,
+             `%s` binary (%s) not null,
              `%s` mediumint unsigned not null,
              `%s` int unsigned not null,
          INDEX (%s),
          UNIQUE KEY `unique_constraint` (%s, %s, %s),
          FOREIGN KEY (%s) REFERENCES %s(%s) ON DELETE CASCADE
     ) ENGINE=INNODB;""" % (
-        FINGERPRINTS_TABLENAME, Database.FIELD_HASH,
+        FINGERPRINTS_TABLENAME, Database.FIELD_HASH, str(math.ceil(FINGERPRINT_REDUCTION/2.)),
         Database.FIELD_SONG_ID, Database.FIELD_OFFSET, Database.FIELD_HASH,
         Database.FIELD_SONG_ID, Database.FIELD_OFFSET, Database.FIELD_HASH,
         Database.FIELD_SONG_ID, SONGS_TABLENAME, Database.FIELD_SONG_ID
@@ -75,11 +78,12 @@ class SQLDatabase(Database):
             `%s` varchar(250) not null,
             `%s` tinyint default 0,
             `%s` binary(20) not null,
+            `%s` float,
         PRIMARY KEY (`%s`),
         UNIQUE KEY `%s` (`%s`)
     ) ENGINE=INNODB;""" % (
         SONGS_TABLENAME, Database.FIELD_SONG_ID, Database.FIELD_SONGNAME, FIELD_FINGERPRINTED,
-        Database.FIELD_FILE_SHA1,
+        Database.FIELD_FILE_SHA1, Database.AUDIO_LENGTH,
         Database.FIELD_SONG_ID, Database.FIELD_SONG_ID, Database.FIELD_SONG_ID,
     )
 
@@ -89,8 +93,8 @@ class SQLDatabase(Database):
             (UNHEX(%%s), %%s, %%s);
     """ % (FINGERPRINTS_TABLENAME, Database.FIELD_HASH, Database.FIELD_SONG_ID, Database.FIELD_OFFSET)
 
-    INSERT_SONG = "INSERT INTO %s (%s, %s) values (%%s, UNHEX(%%s));" % (
-        SONGS_TABLENAME, Database.FIELD_SONGNAME, Database.FIELD_FILE_SHA1)
+    INSERT_SONG = "INSERT INTO %s (%s, %s, %s) values (%%s, UNHEX(%%s), %%s);" % (
+        SONGS_TABLENAME, Database.FIELD_SONGNAME, Database.FIELD_FILE_SHA1, Database.AUDIO_LENGTH)
 
     # selects
     SELECT = """
@@ -107,8 +111,8 @@ class SQLDatabase(Database):
     """ % (Database.FIELD_SONG_ID, Database.FIELD_OFFSET, FINGERPRINTS_TABLENAME)
 
     SELECT_SONG = """
-        SELECT %s, HEX(%s) as %s FROM %s WHERE %s = %%s;
-    """ % (Database.FIELD_SONGNAME, Database.FIELD_FILE_SHA1, Database.FIELD_FILE_SHA1, SONGS_TABLENAME, Database.FIELD_SONG_ID)
+        SELECT %s, HEX(%s) as %s, %s FROM %s WHERE %s = %%s;
+    """ % (Database.FIELD_SONGNAME, Database.FIELD_FILE_SHA1, Database.FIELD_FILE_SHA1, Database.AUDIO_LENGTH, SONGS_TABLENAME, Database.FIELD_SONG_ID)
 
     SELECT_NUM_FINGERPRINTS = """
         SELECT COUNT(*) as n FROM %s
@@ -234,12 +238,12 @@ def insert(self, hash, sid, offset):
         with self.cursor(charset="utf8") as cur:
             cur.execute(self.INSERT_FINGERPRINT, (hash, sid, offset))
 
-    def insert_song(self, songname, file_hash):
+    def insert_song(self, songname, file_hash, audio_length):
         """
         Inserts song in the database and returns the ID of the inserted record.
         """
         with self.cursor(charset="utf8") as cur:
-            cur.execute(self.INSERT_SONG, (songname, file_hash))
+            cur.execute(self.INSERT_SONG, (songname, file_hash, audio_length))
             return cur.lastrowid
 
     def query(self, hash):
@@ -272,34 +276,38 @@ def insert_hashes(self, sid, hashes):
         for hash, offset in hashes:
             values.append((hash, sid, offset))
 
+        base_query = "INSERT IGNORE INTO fingerprints (%s, %s, %s) values " % (Database.FIELD_HASH, Database.FIELD_SONG_ID, Database.FIELD_OFFSET)
         with self.cursor(charset="utf8") as cur:
+            values.sort(key=lambda tup: tup[0])
+            cur.execute("START TRANSACTION;")
             for split_values in grouper(values, 1000):
-                cur.executemany(self.INSERT_FINGERPRINT, split_values)
+                values2tuple = tuple(chain.from_iterable(split_values))
+                query = base_query + ', '.join(['(UNHEX(%s), %s, %s)'] * len(split_values))
+                query += ";"
+                cur.execute(query, values2tuple)
+            cur.execute("COMMIT;")
 
-    def return_matches(self, hashes):
+
+    def return_matches(self, mapper):
         """
         Return the (song_id, offset_diff) tuples associated with
         a list of (sha1, sample_offset) values.
         """
         # Create a dictionary of hash => offset pairs for later lookups
-        mapper = {}
-        for hash, offset in hashes:
-            mapper[hash.upper()] = offset
 
         # Get an iteratable of all the hashes we need
         values = mapper.keys()
 
         with self.cursor(charset="utf8") as cur:
-            for split_values in grouper(values, 1000):
-                # Create our IN part of the query
-                query = self.SELECT_MULTIPLE
-                query = query % ', '.join(['UNHEX(%s)'] * len(split_values))
+            # Create our IN part of the query
+            query = self.SELECT_MULTIPLE
+            query = query % ', '.join(['UNHEX(%s)'] * len(values))
 
-                cur.execute(query, split_values)
+            cur.execute(query, values)
 
-                for hash, sid, offset in cur:
-                    # (sid, db_offset - song_sampled_offset)
-                    yield (sid, offset - mapper[hash])
+            for hash, sid, offset in cur:
+                # (sid, db_offset - song_sampled_offset)
+                yield (sid, offset - mapper[hash])
 
     def __getstate__(self):
         return (self._options,)

dejavu/decoder.py

@@ -72,8 +72,7 @@ def read(filename, limit=None):
         channels = []
         for chn in audiofile:
             channels.append(chn)
-
-    return channels, audiofile.frame_rate, unique_hash(filename)
+    return channels, audiofile.frame_rate, unique_hash(filename), float(len(audiofile))/1000.0
 
 
 def path_to_songname(path):

dejavu/fingerprint.py

@@ -61,6 +61,10 @@
 # with potentially lesser collisions of matches.
 FINGERPRINT_REDUCTION = 20
 
+# SHA1 has 40 hexadecimal chars to encode
+FINGERPRINT_REDUCTION = (40 if FINGERPRINT_REDUCTION > 40 else FINGERPRINT_REDUCTION)
+FINGERPRINT_REDUCTION = (FINGERPRINT_REDUCTION + 1 if FINGERPRINT_REDUCTION % 2 == 1 else FINGERPRINT_REDUCTION)
+
 def fingerprint(channel_samples, Fs=DEFAULT_FS,
                 wsize=DEFAULT_WINDOW_SIZE,
                 wratio=DEFAULT_OVERLAP_RATIO,

dejavu/recognize.py

@@ -14,9 +14,12 @@ def __init__(self, dejavu):
 
     def _recognize(self, *data):
         matches = []
+        total_hashes = 0
         for d in data:
-            matches.extend(self.dejavu.find_matches(d, Fs=self.Fs))
-        return self.dejavu.align_matches(matches)
+            extracted_matches = self.dejavu.find_matches(d, Fs=self.Fs)
+            total_hashes += extracted_matches[1]
+            matches.extend(extracted_matches[0])
+        return self.dejavu.align_matches(matches, total_hashes)
 
     def recognize(self):
         pass  # base class does nothing
@@ -27,7 +30,7 @@ def __init__(self, dejavu):
         super(FileRecognizer, self).__init__(dejavu)
 
     def recognize_file(self, filename):
-        frames, self.Fs, file_hash = decoder.read(filename, self.dejavu.limit)
+        frames, self.Fs, file_hash, audio_length = decoder.read(filename, self.dejavu.limit)
 
         t = time.time()
         match = self._recognize(*frames)