Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor Comments and Explanations added to code #97

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Minor Comments and Explanations added to code #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2229e09
Update core.py
arshkumarsingh Jun 28, 2024
74bd263
Update core.py
arshkumarsingh Jun 28, 2024
ada9fa7
Update core.py
arshkumarsingh Jun 28, 2024
e88b814
Update core.py
arshkumarsingh Jun 28, 2024
8d34756
Update utils.py
arshkumarsingh Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 63 additions & 21 deletions cryptocmd/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,23 @@ def __init__(
id_number=None,
):
"""
:param coin_code: coin code of cryptocurrency e.g. btc. Will be ignored if using id_number.
:param start_date: date since when to scrape data (in the format of dd-mm-yyyy)
:param end_date: date to which scrape the data (in the format of dd-mm-yyyy)
:param all_time: 'True' if need data of all time for respective cryptocurrency
:param order_ascending: data ordered by 'Date' in ascending order (i.e. oldest first).
:param fiat: fiat code eg. USD, EUR
:param coin_name: coin name in case of many coins with same code e.g. sol -> solana, solcoin
:param id_number: id number for the a cryptocurrency on the coinmarketcap.com.
Will override coin_code and coin_name when provided.
Initialize the CmcScraper object.

Args:
coin_code (str): Coin code of the cryptocurrency (e.g. btc).
Will be ignored if using id_number.
start_date (str): Date since when to scrape data (in the format of dd-mm-yyyy).
end_date (str): Date to which scrape the data (in the format of dd-mm-yyyy).
all_time (bool): If True, download data for all time for the respective cryptocurrency.
order_ascending (bool): If True, data is ordered by 'Date' in ascending order (oldest first).
fiat (str): Fiat code (e.g. USD, EUR).
coin_name (str): Coin name in case of multiple coins with the same code
(e.g. sol -> solana, solcoin).
id_number (int): ID number for the cryptocurrency on the CoinMarketCap website.
Overrides coin_code and coin_name when provided.
"""

# Initialize attributes
self.coin_code = coin_code
self.start_date = start_date
self.end_date = end_date
Expand All @@ -70,18 +76,31 @@ def __init__(
self.rows = []
self.id_number = id_number

# enable all_time download if start_time or end_time is not given
# Enable all_time download if start_time or end_time is not given
if not (self.start_date and self.end_date):
self.all_time = True

# Raise error if neither all_time nor start_date and end_date are given
if not (self.all_time or (self.start_date and self.end_date)):
raise InvalidParameters(
"'start_date' or 'end_date' cannot be empty if 'all_time' flag is False"
)

def __repr__(self):
"""
Return string representation of the object.

Returns:
str: String representation of the object.
"""

# Format the string with the object's attributes
return (
"<CmcScraper coin_code:{}, start_date:{}, end_date:{}, all_time:{}>".format(
"<CmcScraper "
"coin_code:{}, "
"start_date:{}, "
"end_date:{}, "
"all_time:{}>".format(
self.coin_code, self.start_date, self.end_date, self.all_time
)
)
Expand Down Expand Up @@ -140,59 +159,75 @@ def _download_data(self, **kwargs):
def get_data(self, format="", verbose=False, **kwargs):
"""
This method returns the downloaded data in specified format.
:param format: extension name of data format. Available: json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods

:param format: Extension name of data format. Available: json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods.
:param verbose: (optional) Flag to enable verbose only.
:param kwargs: Optional arguments that data downloader takes.
:return:
:return: Data in specified format, or if format is not specified, returns headers and rows.
"""

# Download the data
self._download_data(**kwargs)

# If verbose flag is set, print the data
if verbose:
# Print the headers
print(*self.headers, sep=", ")

# Print each row of data
for row in self.rows:
print(*row, sep=", ")
# If format is specified, export the data in that format
elif format:
# Create a new tablib Dataset and add headers
data = tablib.Dataset()
data.headers = self.headers

# Add each row of data to the Dataset
for row in self.rows:
data.append(row)

# Export the Dataset in the specified format and return it
return data.export(format)
# If no format is specified, return the headers and rows
else:
return self.headers, self.rows

def get_dataframe(self, date_as_index=False, **kwargs):
"""
This gives scraped data as DataFrame.
:param date_as_index: make 'Date' as index and remove 'Date' column.
:param kwargs: Optional arguments that data downloader takes.

:param date_as_index: (optional) make 'Date' as index and remove 'Date' column. Default is False.
:param kwargs: (optional) Optional arguments that data downloader takes.
:return: DataFrame of the downloaded data.
"""

try:
import pandas as pd
except ImportError:
pd = None
# Import pandas library
import pandas as pd

# Check if pandas is installed
if pd is None:
raise NotImplementedError(
"DataFrame Format requires 'pandas' to be installed."
"Try : pip install pandas"
)

# Download the data
self._download_data(**kwargs)

# Create a DataFrame from the downloaded data
dataframe = pd.DataFrame(data=self.rows, columns=self.headers)

# convert 'Date' column to datetime type
# Convert 'Date' column to datetime type
dataframe["Date"] = pd.to_datetime(
dataframe["Date"], format="%d-%m-%Y", dayfirst=True
)

# If date_as_index is True, set 'Date' column as index and drop the 'Date' column
if date_as_index:
# set 'Date' column as index and drop the the 'Date' column.
dataframe.set_index("Date", inplace=True)

# Return the DataFrame
return dataframe

def export_csv(self, csv_name=None, csv_path=None, **kwargs):
Expand Down Expand Up @@ -241,13 +276,15 @@ def export_csv(self, csv_name=None, csv_path=None, **kwargs):
def export(self, format, name=None, path=None, **kwargs):
"""
Exports the data to specified file format

:param format: extension name of file format. Available: json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods
:param name: (optional) name of file.
:param path: (optional) output file path.
:param kwargs: Optional arguments that data downloader takes.
:return:
"""

# Get the data in the specified format
data = self.get_data(format, **kwargs)

if path is None:
Expand All @@ -261,18 +298,23 @@ def export(self, format, name=None, path=None, **kwargs):
)

if not name.endswith(".{}".format(format)):
# Add the file extension if not provided
name += ".{}".format(format)

_file = "{0}/{1}".format(path, name)

try:
# Write the data to the file
with open(_file, "wb") as f:
if isinstance(data, str):
# If data is a string, encode it to bytes
f.write(data.encode("utf-8"))
else:
# If data is not a string, simply write it to the file
f.write(data)
except IOError as err:
errno, strerror = err.args
print("I/O error({0}): {1}".format(errno, strerror))
except Exception as err:
# Catch any other exception and print the format and error message
print("format: {0}, Error: {1}".format(format, err))
46 changes: 38 additions & 8 deletions cryptocmd/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,17 @@

def get_url_data(url):
"""
This method downloads the data of the web page.
Downloads the data of the web page.

:param url: 'url' of the web page to download
:return: response object of get request of the 'url'
"""

try:
# Send a GET request to the specified URL and return the response.
response = get(url)
return response
except Exception as e:
# If an exception occurs, print the error message and re-raise the exception.
if hasattr(e, "message"):
print("Error message (get_url_data) :", e.message)
else:
Expand All @@ -29,34 +31,46 @@ def get_url_data(url):
def get_coin_id(coin_code, coin_name):
"""
This method fetches the name(id) of currency from the given code

:param coin_code: coin code of a cryptocurrency e.g. btc
:param coin_name: coin name in case of many coins with same code e.g. sol -> solana, solcoin
:return: coin-id for the a cryptocurrency on the coinmarketcap.com
"""

# Construct the API URL with the given coin code
api_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/map?symbol={coin_code}".format(
coin_code=coin_code
)

try:
# Fetch the JSON data from the API
json_data = get_url_data(api_url).json()

# Check if there was an error in the API response
error_code = json_data["status"]["error_code"]
if error_code == 0:
# If no error, check if coin_name is provided
if coin_name is None:
# If not, return the first data entry's slug
return json_data["data"][0]["slug"]

# If coin_name is provided, filter the data to find the entry with the matching name
return [
data["slug"]
for data in json_data["data"]
if data["name"].lower() == coin_name.lower()
][0]

# If there was an error in the API response, raise an exception
if error_code == 400:
raise InvalidCoinCode(
"'{}' coin code is unavailable on coinmarketcap.com".format(coin_code)
)
else:
raise Exception(json_data["status"]["error_message"])

except Exception as e:
# If an exception occurs, print the error message
print("Error fetching coin id data for coin code {}".format(coin_code))

if hasattr(e, "message"):
Expand All @@ -78,17 +92,19 @@ def download_coin_data(
:param coin_name: coin name in case of many coins with same code e.g. sol -> solana, solcoin
:param id_number: id number for the token on coinmarketcap. Will override coin_code and coin_name when provided.

:return: returns html of the webpage having historical data of cryptocurrency for certain duration
:return: returns json data of the webpage having historical data of cryptocurrency for certain duration
"""

# set default start date if not provided
if start_date is None:
# default start date on coinmarketcap.com
start_date = "28-4-2013"

# set default end date to yesterday if not provided
if end_date is None:
yesterday = datetime.date.today() - datetime.timedelta(1)
end_date = yesterday.strftime("%d-%m-%Y")

# get coin id if not provided
if not id_number:
coin_id = get_coin_id(coin_code, coin_name)

Expand All @@ -108,19 +124,30 @@ def download_coin_data(
.timestamp()
)

# construct the api url
if id_number:
api_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical?convert={}&id={}&time_end={}&time_start={}".format(
fiat, id_number, end_date_timestamp, start_date_timestamp
api_url = (
f"https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical?"
f"convert={fiat}&id={id_number}&time_end={end_date_timestamp}&"
f"time_start={start_date_timestamp}"
)
else:
api_url = "https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical?convert={}&slug={}&time_end={}&time_start={}".format(
fiat, coin_id, end_date_timestamp, start_date_timestamp
api_url = (
f"https://web-api.coinmarketcap.com/v1/cryptocurrency/ohlcv/historical?"
f"convert={fiat}&slug={coin_id}&time_end={end_date_timestamp}&"
f"time_start={start_date_timestamp}"
)

try:
# fetch the json data from the api
json_data = get_url_data(api_url).json()

# check if there was an error in the api response
if json_data["status"]["error_code"] != 0:
raise Exception(json_data["status"]["error_message"])

# if id_number is used, check if coin_code and coin_name are not provided or different
# from the returned data
if id_number:
show_coin_info = False
if coin_code and coin_code != json_data["data"]["symbol"]:
Expand All @@ -140,8 +167,11 @@ def download_coin_data(
f"""The returned data belongs to coin "{json_data['data']['name']}", """
+ f"""with symbol "{json_data['data']['symbol']}" """
)

return json_data

except Exception as e:
# print error message if an exception occurs
print(
"Error fetching price data for {} for interval '{}' and '{}'".format(
f"(id {id_number})" if id_number else coin_code,
Expand Down
Loading