Merge pull request #19 from hechlerp/APIRequests

Add schema and optimizations for api fetcher

api_request/riot_api_fetcher.py

@@ -8,7 +8,6 @@ class MatchFetcher:
 
 
     RANKED_PARAM = "?rankedQueues=TEAM_BUILDER_DRAFT_RANKED_5x5&beginTime="
-    API_KEY = "" #Put your API key here
 
     # These are all of the LoL regions. Deleting from this list and the corresponding
     # SUMMONERS_BY_REGION dictionary will reduce the number of servers being queried.
@@ -29,8 +28,9 @@ class MatchFetcher:
 
 
 
-    def __init__(self):
+    def __init__(self, api_key):
         self.total_matches = {}
+        self.api_key = "api_key=" + api_key
 
     def fetch_matches(self, query_distance, match_number_per_region):
         total_query_distance = int((time.time() * 1000) - query_distance)
@@ -39,7 +39,7 @@ def fetch_matches(self, query_distance, match_number_per_region):
 
         regions = MatchFetcher.REGIONS.keys()
         for region in regions:
-            threads.append(RequestThread(region, region, self.SUMMONERS_BY_REGION[region], total_query_distance, match_number_per_region))
+            threads.append(RequestThread(region, region, self.SUMMONERS_BY_REGION[region], total_query_distance, match_number_per_region, self.api_key))
             self.total_matches[region] = {}
         for thread in threads:
             thread.start()
@@ -51,11 +51,11 @@ def fetch_matches(self, query_distance, match_number_per_region):
         return self.total_matches
 
 
-exitFlag = 0
 
 class RequestThread(threading.Thread):
-    def __init__(self, region, threadName, starting_summoner, query_distance, matches_requested):
+    def __init__(self, region, threadName, starting_summoner, query_distance, matches_requested, api_key):
         threading.Thread.__init__(self)
+        self.api_key = api_key
         self.region = region
         self.name = threadName
         self.request_tracker = deque([])
@@ -106,10 +106,11 @@ def check_helper(self, tracker, api_limit):
     def extract_match_data(self, match_id, summoner_queue):
         regional_url = self.build_regional_url("Match")
         self.track_request()
-        match_data = requests.get(("{0}" + "{1}" + "?" + "{2}").format(regional_url, match_id, MatchFetcher.API_KEY))
+        match_data = requests.get(("{0}" + "{1}" + "?" + "{2}").format(regional_url, match_id, self.api_key))
         print match_data.status_code
         if match_data.status_code == 200:
             match_details = json.loads(match_data.text)
+            self.lockout_counter = 0
             self.total_matches[match_id] = match_details
             for summoner in match_details["participantIdentities"]:
                 summoner_queue.append(summoner["player"]["summonerId"])
@@ -121,25 +122,36 @@ def extract_match_data(self, match_id, summoner_queue):
             if self.check_for_lockout(match_data) == False:
                 return True
 
+        elif match_data.status_code == 403 or match_data.status_code == 404:
+            return True #Break if forbidden. This could be blacklisting or an api key error. Also break on bad input
+
+        elif match_data.status_code == 500 or match_data.status_code == 503:
+            return extract_match_data(match_id, summoner_queue)
+
         return False
 
     def fetch_matches(self, summoner_id = "none"):
         if summoner_id == "none":
             summoner_id = self.starting_summoner
 
             regional_url = self.build_regional_url("Matchlist")
-            request_url = ("{0}" + "{1}" + "{2}" + "{3}" + "&" + "{4}").format(regional_url, summoner_id, MatchFetcher.RANKED_PARAM, self.query_distance, MatchFetcher.API_KEY)
+            request_url = ("{0}" + "{1}" + "{2}" + "{3}" + "&" + "{4}").format(regional_url, summoner_id, MatchFetcher.RANKED_PARAM, self.query_distance, self.api_key)
             self.track_request()
             matches = requests.get(request_url)
             print matches.status_code
             if matches.status_code == 200:
+                self.lockout_counter = 0
                 self.summoners[summoner_id] = True
                 return json.loads(matches.text)["matches"]
 
             elif matches.status_code == 429: #break if getting rate limited. Don't want to get blacklisted!
                 if self.check_for_lockout(matches) == False:
                     return False
-                else: self.fetch_matches(summoner_id)
+
+            elif matches.status_code == 403 or matches.status_code == 404:
+                return False
+
+
 
     def check_for_lockout(self, response):
         print response.headers
@@ -156,14 +168,13 @@ def check_for_lockout(self, response):
 
         return True
 
-
     def fetcher(self):
         finished = False
         match_queue = self.set_up_match_queue()
         summoner_queue = deque([])
+        if len(match_queue) == 0:
+            return
         while finished == False:
-            if exitFlag:
-                self.name.exit()
             self.halt_requests()
             if len(match_queue) > 0:
                 match_id = match_queue.popleft()
@@ -188,22 +199,24 @@ def set_up_match_queue(self):
         if initial_matches == False:
             return match_queue
         while initial_matches is None:
+            self.halt_requests()
             initial_matches = self.fetch_matches()
-        for match in initial_matches:
-            match_queue.append(match["matchId"])
+        if initial_matches is not None:
+            for match in initial_matches:
+                match_queue.append(match["matchId"])
 
         return match_queue
 
 
 
 
 # For testing a single thread
-# a = RequestThread("NA", "NA", MatchFetcher.SUMMONERS_BY_REGION["NA"] )
+# a = RequestThread("NA", "NA", MatchFetcher.SUMMONERS_BY_REGION["NA"], 1464039831476, 10, API_KEY)
 # a.start()
 # a.join()
-
+#
 # For testing the fetcher class
-# a = MatchFetcher()
+# a = MatchFetcher(API_KEY)
 # results = a.fetch_matches(1209600000, 10) # two weeks ago in milliseconds, finding 10 matches per region
 # for region in results.keys():
 #     print ("Matches for {0}:").format(region)

api_request/testfile.py

@@ -2,11 +2,12 @@
 import time
 from riot_api_fetcher import *
 
+#To run the tests, put your API Key in the places where they are commented out
 class TestRiotApi(unittest.TestCase):
 
     def setUp(self):
-        self.test_thread = RequestThread("NA", "NA", MatchFetcher.SUMMONERS_BY_REGION["NA"], int((time.time() * 1000) - 1209600000), 10)
-        self.test_fetcher = MatchFetcher()
+        self.test_thread = RequestThread("NA", "NA", MatchFetcher.SUMMONERS_BY_REGION["NA"], int((time.time() * 1000) - 1209600000), 10, #API Key)
+        self.test_fetcher = MatchFetcher(#API Key)
 
 class BasicApiTest(TestRiotApi):
     def test_thread_init_region(self):
@@ -32,7 +33,14 @@ def test_fetcher_run(self):
         results = self.test_fetcher.fetch_matches(int((time.time() * 1000) - 1209600000), 1)
         self.assertEqual(len(results), 11)
 
-
+    def test_fetcher_bad_input(self):
+        bad_thread = RequestThread("NA", "NA", MatchFetcher.SUMMONERS_BY_REGION["NA"] + "00", int((time.time() * 1000) - 1209600000), 1, #API Key)
+        bad_thread.start()
+        bad_thread.join()
+        results = bad_thread.total_matches
+        self.assertEqual(len(results), 0)
+
+
 
 if __name__ == '__main__':
     suite1 = unittest.TestLoader().loadTestsFromTestCase(BasicApiTest)

schema.md

@@ -0,0 +1,32 @@
+# Schema Outline
+
+
+#DB 1 (raw)
+
+## Champions_in_matches
+column name | data type | details
+------------|-----------|-----------------------
+id          | integer   | not null, primary key
+champion_id | string    | not null, foreign key(references Champions)
+match_id    | string    | not null
+winner      | boolean   |
+team        | string    | not null
+role        | string    |
+
+#DB 2 (prepped)
+
+## Champions
+column name      | data type | details
+-----------------|-----------|-----------------------
+id               | integer   | not null, primary key
+name             | string    | not null
+pick/ban rate    | integer   |
+win rate         | integer   |
+metascore        | integer   |
+top              | integer   | not null
+mid              | integer   | not null
+jungle           | integer   | not null
+marksman         | integer   | not null
+support          | integer   | not null
+
+image            | img       | not null