Coverage for src/scrilla/services.py: 73%

1# This file is part of scrilla: https://github.com/chinchalinchin/scrilla.

3# scrilla is free software: you can redistribute it and/or modify

4# it under the terms of the GNU General Public License version 3

5# as published by the Free Software Foundation.

7# scrilla is distributed in the hope that it will be useful,

8# but WITHOUT ANY WARRANTY; without even the implied warranty of

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

10# GNU General Public License for more details.

12# You should have received a copy of the GNU General Public License

13# along with scrilla. If not, see <https://www.gnu.org/licenses/>

14# or <https://github.com/chinchalinchin/scrilla/blob/develop/main/LICENSE>.

16"""

17This module interfaces with the external services the program uses to hydrate with financial data. In the case of price and interest history, the functions in this module defer to the cache before making expensive HTTP requests. Statistical data retrieved from FRED and dividment payment histories are not persisted in the cache since most of the data is reported on an irregular basis and it is impossible to tell based on the date alone whether or not the cache is out of date.

19This module can be imported and used directly in other Python scripts if the API keys for the services have been set.

21```

22import os

23from scrilla.services import get_daily_price_history

25os.environ.setdefault('ALPHA_VANTAGE_KEY')

26prices = get_daily_price_history('AAPL')

27```

28"""

29import itertools

30import time

31import requests

32from typing import Dict, List, Union

33import defusedxml.ElementTree as ET

35from datetime import date

37from scrilla import settings, cache

38from scrilla.static import keys, constants

39from scrilla.util import errors, outputter, helper, dater

41logger = outputter.Logger("scrilla.services", settings.LOG_LEVEL)

44class StatManager():

45 """

46 StatManager is an interface between the application and the external services that hydrate it with financial statistics data. This class gets instantiated on the level of the `scrilla.services` module with the value defined in `scrilla.settings.STAT_MANAGER`. This value is in turn defined by the value of the `STAT_MANAGER` environment variable. This value determines how the url is constructed, which API credentials get appended to the external query and the keys used to parse the response JSON containing the statistical data.

48 Attributes

49 ----------

50 1. **genre**: ``str``

51 A string denoting which service will be used for data hydration. Genres can be accessed through the `keys.keys['SERVICES']` dictionary.

52 2. **self.service_map**: ``dict``

53 A dictionary containing keys unique to the service defined by `genre`, such as endpoints, query parameters, etc.

55 Raises

56 ------

57 1. **scrilla.errors.ConfigurationError**

58 If the **STAT_MANAGER** environment variable has been set to a value the program doesn't understand or hasn't set at all, this error will be thrown when this class is instantiated.

60 .. notes ::

61 * This class handles retrieval of financial statistics from the [Quandl's FRED dataset](https://data.nasdaq.com/data/FRED-federal-reserve-economic-data) and [Quandl's USTREASURY dataset](https://data.nasdaq.com/data/USTREASURY-us-treasury). In other words, it handles statistics and interest rate service calls. Interest rates are technically prices, not statistics, but...well, I don't have a good explanation why this isn't `scrilla.services.PriceManager`; I suppose I grouped these classes more by service type than data type. Might do to refactor...

62 """

64 def __init__(self, genre):

65 self.genre = genre

66 self.service_map = None

67 if self._is_quandl(): 67 ↛ 68line 67 didn't jump to line 68, because the condition on line 67 was never true

68 self.service_map = keys.keys["SERVICES"]["STATISTICS"]["QUANDL"]["MAP"]

69 self.key = settings.Q_KEY

70 self.url = settings.Q_URL

71 elif self._is_treasury(): 71 ↛ 75line 71 didn't jump to line 75, because the condition on line 71 was never false

72 self.service_map = keys.keys["SERVICES"]["STATISTICS"]["TREASURY"]["MAP"]

73 self.url = settings.TR_URL

74 self.key = None

75 if self.service_map is None: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 raise errors.ConfigurationError(

77 'No STAT_MANAGER found in the environment settings')

79 def _is_quandl(self):

80 """

81 Returns

82 -------

83 `bool`

84 `True` if this instace of `StatManager` is a Quandl interface. `False` otherwise.

86 .. notes::

87 * This is for use within the class and probably won't need to be accessed outside of it. `StatManager` is intended to hide the data implementation from the rest of the library, i.e. it is ultimately agnostic about where the data comes where. It should never need to know `StatManger` is a Quandl interface. Just in case the library ever needs to populate its data from another source.

89 """

90 if self.genre == keys.keys['SERVICES']['STATISTICS']['QUANDL']['MANAGER']: 90 ↛ 91line 90 didn't jump to line 91, because the condition on line 90 was never true

91 return True

92 return False

94 def _is_treasury(self):

95 """

96 Returns

97 -------

98 `bool`

99 `True` if this instace of `StatManager` is a US Treasury interface. `False` otherwise.

100

101 .. notes::

102 * This is for use within the class and probably won't need to be accessed outside of it. `StatManager` is intended to hide the data implementation from the rest of the library, i.e. it is ultimately agnostic about where the data comes where. It should never need to know `StatManger` is a Quandl interface. Just in case the library ever needs to populate its data from another source.

103

104 """

105 if self.genre == keys.keys['SERVICES']['STATISTICS']['TREASURY']['MANAGER']: 105 ↛ 107line 105 didn't jump to line 107, because the condition on line 105 was never false

106 return True

107 return False

108

109 def _construct_query(self, start_date: date, end_date: date) -> str:

110 """

111 Constructs and formats the query parameters for the external statistics service. Note, this method appends the API key to the query. Be careful with the returned value.

112

113 Parameters

114 ----------

115 1. **start_date** : ``datetime.date``

116 Start date of historical sample to be retrieved.

117 2. **end_date** : ``datetime.date``

118 End date of historical sample to be retrieved.

119

120 Returns

121 -------

122 ``str``

123 The formatted query for the specific service defined by `self.genre`.

124

125 """

126 query = ""

127

128 if end_date is not None: 128 ↛ 135line 128 didn't jump to line 135, because the condition on line 128 was never false

129 if self._is_treasury(): 129 ↛ 132line 129 didn't jump to line 132, because the condition on line 129 was never false

130 end_string = "all"

131 else:

132 end_string = dater.to_string(end_date)

133 query += f'&{self.service_map["PARAMS"]["END"]}={end_string}'

134

135 if start_date is not None and not self._is_treasury(): 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true

136 start_string = dater.to_string(start_date)

137 query += f'&{self.service_map["PARAMS"]["START"]}={start_string}'

138

139 logger.debug(

140 f'StatManager Query (w/o key) = {query}', 'StatManager._construct_query')

141

142 if self.service_map["PARAMS"].get("KEY", None) is not None: 142 ↛ 143line 142 didn't jump to line 143, because the condition on line 142 was never true

143 if query:

144 return f'{query}&{self.service_map["PARAMS"]["KEY"]}={self.key}'

145 return f'{self.service_map["PARAMS"]["KEY"]}={self.key}'

146 return query

147

148 def _construct_stat_url(self, symbol: str, start_date: date, end_date: date):

149 """

150 Constructs the full URL path for the external statistics service. Note, this method will return the URL with an API key appended as a query parameter. Be careful with the returned value.

151

152 Parameters

153 ----------

154 1. **symbol**: ``str``

155 Symbol representing the statistical series to be retrieved. List of allowable symbols can be found [here](https://data.nasdaq.com/data/FRED-federal-reserve-economic-data)

156 2. **start_date**: ``datetime.date``

157 Start date of historical sample to be retrieved.

158 3. **end_date**: ``datetime.date``

159 End date of historical sample to be retrieved.

160

161

162 Returns

163 -------

164 ``str``

165 The formatted URL for the specific statistics service defined by `self.genre`.

166 """

167 url = f'{self.url}/{self.service_map["PATHS"]["FRED"]}/{symbol}?'

168 url += self._construct_query(start_date=start_date, end_date=end_date)

169 return url

170

171 def _construct_interest_url(self, start_date, end_date):

172 """

173 Constructs the full URL path for the external interest rate service. Note, this method will return the URL with an API key appended as a query parameter. Be careful with the returned value.

174

175 Parameters

176 ----------

177 1. **start_date**: ``datetime.date``

178 Start date of historical sample to be retrieved.

179 2. **end_date**: ``datetime.date``

180 End date of historical sample to be retrieved.

181

182 Returns

183 -------

184 ``str``

185 The formatted URL for the specific interest rate service defined by `self.genre`.

186

187 .. notes::

188 * The URL returned by this method will always contain a query for a historical range of US Treasury Yields, i.e. this method is specifically for queries involving the "Risk-Free" (right? right? *crickets*) Yield Curve.

189 """

190 url = f'{self.url}/{self.service_map["PATHS"]["YIELD"]}?'

191 if self._is_treasury(): 191 ↛ 193line 191 didn't jump to line 193, because the condition on line 191 was never false

192 url += f'{self.service_map["PARAMS"]["DATA"]}={self.service_map["ARGUMENTS"]["DAILY"]}'

193 url += self._construct_query(start_date=start_date, end_date=end_date)

194 return url

195

196 def get_stats(self, symbol, start_date, end_date):

197 url = self._construct_stat_url(symbol, start_date, end_date)

198 response = requests.get(url).json()

199

200 raw_stat = response[self.service_map["KEYS"]["FIRST_LAYER"]

201 ][self.service_map["KEYS"]["SECOND_LAYER"]]

202 formatted_stat = {}

203

204 for stat in raw_stat:

205 formatted_stat[stat[0]] = stat[1]

206 return formatted_stat

207

208 def get_interest_rates(self, start_date, end_date):

209 """

210

211 .. notes::

212 - Regardless of the `scrilla.settings.STAT_MANAGER`, the return format for this method is as follows:

213 ```json

214 {

215 "date": [ "value", "value", ... , "value" ],

216 "date": [ "value", "value", ... , "value" ]

217 }

218 ```

219 """

220 url = self._construct_interest_url(

221 start_date=start_date, end_date=end_date)

222 formatted_interest = {}

223

224 if self._is_quandl(): 224 ↛ 225line 224 didn't jump to line 225, because the condition on line 224 was never true

225 response = requests.get(url)

226

227 response = response.json()

228 raw_interest = response[self.service_map["KEYS"]

229 ["FIRST_LAYER"]][self.service_map["KEYS"]["SECOND_LAYER"]]

230 for rate in raw_interest:

231 formatted_interest[rate[0]] = rate[1:]

232

233 elif self._is_treasury(): 233 ↛ 281line 233 didn't jump to line 281, because the condition on line 233 was never false

234 # NOTE: this is ugly, but it's the government's fault for not supporting an API

235 # from this century.

236

237 def __paginate(page_no, page_url):

238 page_url = f'{page_url}&{self.service_map["PARAMS"]["PAGE"]}={page_no}'

239 logger.verbose(

240 f'Paginating: {page_url}', 'StatManager.get_interest_rates.__paginate')

241 page_response = ET.fromstring(requests.get(page_url).text)

242 return page_no - 1, page_response

243

244 record_time = dater.business_days_between(

245 constants.constants['YIELD_START_DATE'], end_date, True)

246 # NOTE: subtract to reindex to 0

247 pages = record_time // self.service_map["KEYS"]["PAGE_LENGTH"] - 1

248 pages += 1 if record_time % self.service_map["KEYS"]["PAGE_LENGTH"] > 0 else 0

249 page = pages

250

251 logger.debug(

252 f'Sorting through {pages} pages of Treasury data', 'StatManager.get_interest_rates')

253 logger.debug(

254 f'Days from {dater.to_string(end_date)} to start of Treasury record: {record_time}', 'StatManager.get_interest_rates')

255

256 while True:

257 page, response = __paginate(page, url)

258 first_layer = response.findall(

259 self.service_map["KEYS"]["FIRST_LAYER"])

260

261 if page >= 0: 261 ↛ 279line 261 didn't jump to line 279, because the condition on line 261 was never false

262 for child in first_layer: 262 ↛ 256line 262 didn't jump to line 256, because the loop on line 262 didn't complete

263 xpath = f'{self.service_map["KEYS"]["RATE_XPATH"]}{self.service_map["KEYS"]["DATE"]}'

264 this_date = dater.parse(child.find(xpath).text)

265

266 if start_date <= this_date <= end_date:

267 date_string = dater.to_string(this_date)

268 formatted_interest[date_string] = []

269

270 for maturity in self.service_map["YIELD_CURVE"].values():

271 interest = child.find(

272 f'{self.service_map["KEYS"]["RATE_XPATH"]}{maturity}').text

273 formatted_interest[date_string].append(

274 float(interest))

275

276 if len(formatted_interest) >= dater.business_days_between(start_date, end_date, True):

277 return formatted_interest

278 else:

279 break

280

281 return formatted_interest

282

283 @staticmethod

284 def format_for_maturity(maturity, results):

285 try:

286 maturity_key = keys.keys['YIELD_CURVE'].index(maturity)

287 except KeyError:

288 raise errors.InputValidationError(

289 f'{maturity} is not a valid maturity for US Treasury Bonds')

290

291 formatted_interest = {}

292 for result in results:

293 formatted_interest[result] = results[result][maturity_key]

294 return formatted_interest

295

296

297class DividendManager():

298 """

299 Attributes

300 ----------

301 1. **genre**: ``str``

302 A string denoting which service will be used for data hydration. Genres can be accessed through the `keys.keys['SERVICES']` dictionary.

303 2. **self.service_map**: ``dict``

304 A dictionary containing keys unique to the service defined by `genre`, such as endpoints, query parameters, etc.

305 """

306

307 def __init__(self, genre):

308 self.genre = genre

309 if self.genre == keys.keys['SERVICES']['DIVIDENDS']['IEX']['MANAGER']: 309 ↛ 314line 309 didn't jump to line 314, because the condition on line 309 was never false

310 self.service_map = keys.keys['SERVICES']['DIVIDENDS']['IEX']['MAP']

311 self.key = settings.iex_key()

312 self.url = settings.IEX_URL

313

314 if self.service_map is None: 314 ↛ 315line 314 didn't jump to line 315, because the condition on line 314 was never true

315 raise errors.ConfigurationError(

316 'No DIV_MANAGER found in the parsed environment settings')

317

318 def _construct_url(self, ticker):

319 query = f'{ticker}/{self.service_map["PATHS"]["DIV"]}/{self.service_map["PARAMS"]["FULL"]}'

320 url = f'{self.url}/{query}?{self.service_map["PARAMS"]["KEY"]}={self.key}'

321 logger.debug(

322 f'DivManager Query (w/o key) = {query}', 'DividendManager._construct_url')

323 return url

324

325 def get_dividends(self, ticker):

326 url = self._construct_url(ticker)

327 response = requests.get(url).json()

328 formatted_response = {}

329

330 for item in response:

331 this_date = str(item[self.service_map['KEYS']['DATE']])

332 div = item[self.service_map['KEYS']['AMOUNT']]

333 formatted_response[this_date] = div

334

335 return formatted_response

336

337

338class PriceManager():

339 """

340 PriceManager is an interface between the application and the external services that hydrate it with price data. This class gets instantiated on the level of the `scrilla.services` module with the value defined in `scrilla.settings.PRICE_MANAGER` variable. This value is in turn configured by the value of the **PRICE_MANAGER** environment variable. This value determines how the url is constructed, which API credentials get appended to the external query and the keys used to parse the response JSON containing the price data.

341

342 Raises

343 ------

344 1. **scrilla.errors.ConfigurationError**

345 If the **PRICE_MANAGER** environment variable hasn't been set or set to a value the program doesn't understand, this error will be thrown when this class is instantiated.

346

347 Attributes

348 ----------

349 1. **genre**: ``str``

350 A string denoting which service will be used for data hydration. Genres can be accessed through the `keys.keys['SERVICES']` dictionary.

351 2. **self.service_map**: ``dict``

352 A dictionary containing keys unique to the service defined by `genre`, such as endpoints, query parameters, etc.

353

354 """

355

356 def __init__(self, genre):

357 self.genre = genre

358 if self.genre == keys.keys['SERVICES']['PRICES']['ALPHA_VANTAGE']['MANAGER']: 358 ↛ 362line 358 didn't jump to line 362, because the condition on line 358 was never false

359 self.service_map = keys.keys['SERVICES']['PRICES']['ALPHA_VANTAGE']['MAP']

360 self.url = settings.AV_URL

361 self.key = settings.av_key()

362 if self.service_map is None: 362 ↛ 363line 362 didn't jump to line 363, because the condition on line 362 was never true

363 raise errors.ConfigurationError(

364 'No PRICE_MANAGER found in the parsed environment settings')

365

366 def _construct_url(self, ticker, asset_type):

367 """

368 Constructs the service url with the query and parameters appended.

369

370 Parameters

371 ----------

372 1. **ticker**: ``str`

373 Ticker symbol of the asset whose prices are being retrieved.

374 2. **asset_type**: ``str``

375 Asset type of the asset whose prices are being retrieved. Options are statically

376 accessible in the `scrillla.static` module dictionary `scrilla.keys.keys['ASSETS']`.

377

378 Returns

379 -------

380 `str`

381 The URL with the authenticated query appended, i.e. with the service's API key injected into the parameters. Be careful not to expose the return value of this function!

382

383 .. notes::

384 * this function will probably need substantially refactored if another price service is ever incorporated, unless the price service selected can be abstracted into the same template set by `scrilla.statics.keys['SERVICES']['PRICES']['ALPHA_VANTAGE']['MAP']`.

385 """

386

387 query = f'{self.service_map["PARAMS"]["TICKER"]}={ticker}'

388

389 if asset_type == keys.keys['ASSETS']['EQUITY']:

390 query += f'&{self.service_map["PARAMS"]["FUNCTION"]}={self.service_map["ARGUMENTS"]["EQUITY_DAILY"]}'

391 query += f'&{self.service_map["PARAMS"]["SIZE"]}={self.service_map["ARGUMENTS"]["FULL"]}'

392

393 elif asset_type == keys.keys['ASSETS']['CRYPTO']: 393 ↛ 397line 393 didn't jump to line 397, because the condition on line 393 was never false

394 query += f'&{self.service_map["PARAMS"]["FUNCTION"]}={self.service_map["ARGUMENTS"]["CRYPTO_DAILY"]}'

395 query += f'&{self.service_map["PARAMS"]["DENOMINATION"]}={constants.constants["DENOMINATION"]}'

396

397 auth_query = query + f'&{self.service_map["PARAMS"]["KEY"]}={self.key}'

398 url = f'{self.url}?{auth_query}'

399 logger.debug(

400 f'PriceManager query (w/o key) = {query}', 'PriceManager._construct_url')

401 return url

402

403 def get_prices(self, ticker: str, start_date: date, end_date: date, asset_type: str):

404 """

405 Retrieve prices from external service.

406

407 Parameters

408 ----------

409 1. **ticker** : ``str``

410 Ticker symbol of the asset whose prices are being retrieved.

411 2. **start_date**; ``str``

412 3. **end_date**: ``str``

413 4. **asset_type** : ``str``

414 Asset type of the asset whose prices are being retrieved. Options are statically

415 accessible in the `scrillla.static` module dictionary `scrilla.keys.keys['ASSETS']`.

416

417 Returns

418 -------

419 ``Dict[str, Dict[str, float]]``

420 ```

421 prices = {

422 'date': {

423 'open': value,

424 'close': value

425 },

426 'date': {

427 'open': value,

428 'close': value

429 }

430 }

431 ```

432 Dictionary of prices with date as key, ordered from latest to earliest.

433

434 Raises

435 ------

436 1. **scrilla.errors.ConfigurationError**

437 If one of the settings is improperly configured or one of the environment variables was unable to be parsed from the environment, this error will be thrown.

438 2. **scrilla.errors.APIResponseError**

439 If the service from which data is being retrieved is down, the request has been rate limited or some otherwise anomalous event has taken place, this error will be thrown.

440 """

441 url = self._construct_url(ticker, asset_type)

442 response = requests.get(url).json()

443

444 first_element = helper.get_first_json_key(response)

445 # end function is daily rate limit is reached

446 if first_element == self.service_map['ERRORS']['RATE_LIMIT']: 446 ↛ 447line 446 didn't jump to line 447, because the condition on line 446 was never true

447 raise errors.APIResponseError(

448 response[self.service_map['ERRORS']['RATE_LIMIT']])

449 # check for bad response

450 if first_element == self.service_map['ERRORS']['INVALID']: 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true

451 raise errors.APIResponseError(

452 response[self.service_map['ERRORS']['INVALID']])

453

454 # check and wait for API rate limit refresh

455 first_pass, first_element = True, helper.get_first_json_key(response)

456

457 while first_element == self.service_map['ERRORS']['RATE_THROTTLE']: 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true

458 if first_pass:

459 logger.info(

460 f'{self.genre} API rate limit per minute exceeded. Waiting...', 'PriceManager.get_prices')

461 first_pass = False

462 else:

463 logger.info('Waiting...', 'PriceManager.get_prices')

464

465 time.sleep(constants.constants['BACKOFF_PERIOD'])

466 response = requests.get(url).json()

467 first_element = helper.get_first_json_key(response)

468

469 if first_element == self.service_map['ERRORS']['INVALID']:

470 raise errors.APIResponseError(

471 response[self.service_map['ERRORS']['INVALID']])

472

473 prices = self._slice_prices(

474 start_date=start_date, end_date=end_date, asset_type=asset_type, prices=response)

475 format_prices = {}

476 for this_date in prices:

477 close_price = self._parse_price_from_date(prices=prices, this_date=this_date, asset_type=asset_type,

478 which_price=keys.keys['PRICES']['CLOSE'])

479 open_price = self._parse_price_from_date(prices=prices, this_date=this_date, asset_type=asset_type,

480 which_price=keys.keys['PRICES']['OPEN'])

481 format_prices[this_date] = {

482 keys.keys['PRICES']['OPEN']: float(open_price), keys.keys['PRICES']['CLOSE']: float(close_price)}

483 return format_prices

484

485 def _slice_prices(self, start_date: date, end_date: date, asset_type: str, prices: dict) -> dict:

486 """

487 Parses the raw response from the external price service into a format the program will understand.

488

489 Parameters

490 ----------

491 1. **start_date** : ``datetime.date``

492 2. **end_date** : ``datetime.date``

493 3. **asset_type** : ``str``

494 Required: Asset type of the asset whose prices are being retrieved. Options are statically

495 accessible in the `scrillla.static` module dictionary `scrilla.keys.keys['ASSETS']`.

496 4. **response** : ``dict``

497 The full response from the price manager, i.e. the entire price history returned by the external service in charge of retrieving pricce histories, the result returned from `scrilla.services.PriceManager.get_prices`

498

499 Returns

500 -------

501 ``dict``: `{ 'date': value, 'date': value, ...}`

502 Dictionary of prices with date as key, ordered from latest to earliest.

503

504

505 Raises

506 ------

507 1. **KeyError**

508 If the inputted or validated dates do not exist in the price history, a KeyError will be thrown. This could be due to the equity not having enough price history, i.e. it started trading a month ago and doesn't have 100 days worth of prices yet, or some other anomalous event in an equity's history.

509 2. **scrilla.errors.ConfigurationError**

510 If one of the settings is improperly configured or one of the environment variables was unable to be parsed from the environment, this error will be thrown.

511 """

512

513 # NOTE: only really needed for `alpha_vantage` responses so far, due to the fact AlphaVantage either returns everything or 100 days of prices.

514 # shouldn't need to verify genre anyway, since using service_map should abstract the response away (hopefully).

515 if self.genre == keys.keys['SERVICES']['PRICES']['ALPHA_VANTAGE']['MANAGER']: 515 ↛ 532line 515 didn't jump to line 532, because the condition on line 515 was never false

516

517 # NOTE: Remember AlphaVantage is ordered current to earliest. END_INDEX is

518 # actually the beginning of slice and START_INDEX is actually end of slice.

519 start_string, end_string = dater.to_string(

520 start_date), dater.to_string(end_date)

521 if asset_type == keys.keys['ASSETS']['EQUITY']:

522 response_map = self.service_map['KEYS']['EQUITY']['FIRST_LAYER']

523 elif asset_type == keys.keys['ASSETS']['CRYPTO']: 523 ↛ 526line 523 didn't jump to line 526, because the condition on line 523 was never false

524 response_map = self.service_map['KEYS']['CRYPTO']['FIRST_LAYER']

525

526 start_index = list(prices[response_map].keys()).index(start_string)

527 end_index = list(prices[response_map].keys()).index(end_string)

528 prices = dict(itertools.islice(

529 prices[response_map].items(), end_index, start_index+1))

530 return prices

531

532 raise errors.ConfigurationError(

533 'No PRICE_MANAGER found in the parsed environment settings')

534

535 def _parse_price_from_date(self, prices: Dict[str, Dict[str, float]], this_date: date, asset_type: str, which_price: str) -> str:

536 """

537 Parameters

538 ----------

539 1. **prices** : ``Dict[str, Dict[str, float]]``

540 List containing the AlphaVantage response with the first layer peeled off, i.e.

541 no metadata, just the date and prices.

542 2. **date**: `date``

543 String of the date to be parsed. Note: this is not a datetime.date object. String

544 must be formatted `YYYY-MM-DD`

545 3. **asset_type**: ``str``

546 String that specifies what type of asset price is being parsed. Options are statically

547 accessible in the `scrillla.static` module dictionary `scrilla.keys.keys['ASSETS']`

548 4. **which_price**: ``str``

549 String that specifies which price is to be retrieved, the closing price or the opening prices. Options are statically accessible

550

551 Returns

552 ------

553 ``str``

554 String containing the price on the specified date.

555

556 Raises

557 ------

558 1. **KeyError**

559 If the inputted or validated dates do not exist in the price history, a KeyError will be thrown. This could be due to the equity not having enough price history, i.e. it started trading a month ago and doesn't have 100 days worth of prices yet, or some other anomalous event in an equity's history.

560 2. **scrilla.errors.InputValidationError**

561 If prices was unable to be grouped into a (crypto, equity)-asset class or the opening/closing price did not exist for whatever reason, this error will be thrown.

562 """

563 if asset_type == keys.keys['ASSETS']['EQUITY']:

564 if which_price == keys.keys['PRICES']['CLOSE']:

565 return prices[this_date][self.service_map['KEYS']['EQUITY']['CLOSE']]

566 if which_price == keys.keys['PRICES']['OPEN']: 566 ↛ 575line 566 didn't jump to line 575, because the condition on line 566 was never false

567 return prices[this_date][self.service_map['KEYS']['EQUITY']['OPEN']]

568

569 elif asset_type == keys.keys['ASSETS']['CRYPTO']: 569 ↛ 575line 569 didn't jump to line 575, because the condition on line 569 was never false

570 if which_price == keys.keys['PRICES']['CLOSE']:

571 return prices[this_date][self.service_map['KEYS']['CRYPTO']['CLOSE']]

572 if which_price == keys.keys['PRICES']['OPEN']: 572 ↛ 575line 572 didn't jump to line 575, because the condition on line 572 was never false

573 return prices[this_date][self.service_map['KEYS']['CRYPTO']['OPEN']]

574

575 raise errors.InputValidationError(

576 f'Verify {asset_type}, {which_price} are allowable values')

577

578

579def get_daily_price_history(ticker: str, start_date: Union[None, date] = None, end_date: Union[None, date] = None, asset_type: Union[None, str] = None) -> Dict[str, Dict[str, float]]:

580 """

581 Wrapper around external service request for price data. Relies on an instance of `PriceManager` configured by `settings.PRICE_MANAGER` value, which in turn is configured by the `PRICE_MANAGER` environment variable, to hydrate with data.

582

583 Before deferring to the `PriceManager` and letting it call the external service, however, this function checks if response is in local cache. If the response is not in the cache, it will pass the request off to `PriceManager` and then save the response in the cache so subsequent calls to the function can bypass the service request. Used to prevent excessive external HTTP requests and improve the performance of the application. Other parts of the program should interface with the external price data services through this function to utilize the cache functionality.

584

585 Parameters

586 ----------

587 1. **ticker** : ``str``

588 Ticker symbol corresponding to the price history to be retrieved.

589 2. **start_date** : ``datetime.date``

590 *Optional*. Start date of price history. Defaults to None. If `start_date is None`, the calculation is made as if the `start_date` were set to 100 trading days ago. If `scrilla.files.get_asset_type(ticker)==scrill.keys.keys['ASSETS']['CRYPTO']`, this includes weekends and holidays. If `scrilla.files.get_asset_type(ticker)==scrilla.keys.keys['ASSETS']['EQUITY']`, this excludes weekends and holidays.

591 3. **end_date** : ``datetime.date``

592 Optional End date of price history. Defaults to None. If `end_date is None`, the calculation is made as if the `end_date` were set to today. If `scrilla.files.get_asset_type(ticker)==scrill.keys.keys['ASSETS']['CRYPTO']`, this means today regardless. If `scrilla.files.get_asset_type(ticker)==scrilla.keys.keys['ASSETS']['EQUITY']`, this excludes weekends and holidays so that `end_date` is set to the previous business date.

593 4. **asset_type** : ``string``

594 *Optional*. Asset type of the ticker whose history is to be retrieved. Used to prevent excessive calls to IO and list searching. `asset_type` is determined by comparing the ticker symbol `ticker` to a large static list of ticker symbols maintained in installation directory's /data/static/ subdirectory, which can slow the program down if the file is constantly accessed and lots of comparison are made against it. Once an `asset_type` is calculated, it is best to preserve it in the process environment somehow, so this function allows the value to be passed in. If no value is detected, it will make a call to the aforementioned directory and parse the file to determine to the `asset_type`. Asset types are statically accessible through the `scrilla.keys.keys['ASSETS']` dictionary.

595

596 Returns

597 ------

598 ``Dict[str, Dict[str, float]]`` : Dictionary with date strings formatted `YYYY-MM-DD` as keys and a nested dictionary containing the 'open' and 'close' price as values. Ordered from latest to earliest, e.g.,

599 ```

600 {

601 'date' :

602 {

603 'open': value,

604 'close': value

605 },

606 'date':

607 {

608 'open' : value,

609 'close' : value

610 },

611 ...

612 }

613 ```

614

615 Raises

616 ------

617 1. **scrilla.errors.PriceError**

618 If no sample prices can be retrieved, this error is thrown.

619

620 .. notes::

621 * The default analysis period, if no `start_date` and `end_date` are specified, is determined by the *DEFAULT_ANALYSIS_PERIOD** variable in the `settings,py` file. The default value of this variable is 100.

622 """

623 asset_type = errors.validate_asset_type(ticker, asset_type)

624 start_date, end_date = errors.validate_dates(

625 start_date, end_date, asset_type)

626

627 cached_prices = price_cache.filter(

628 ticker=ticker, start_date=start_date, end_date=end_date)

629

630 if cached_prices is not None:

631 if asset_type == keys.keys['ASSETS']['EQUITY']:

632 logger.debug(

633 f'Comparing cache-size({len(cached_prices)}) = date-length{dater.business_days_between(start_date, end_date)})', 'get_daily_price_history')

634 elif asset_type == keys.keys['ASSETS']['CRYPTO']: 634 ↛ 639line 634 didn't jump to line 639, because the condition on line 634 was never false

635 logger.debug(

636 f'Comparing cache-size({len(cached_prices)}) = date-length({dater.days_between(start_date, end_date)})', 'get_daily_price_history')

637

638 # make sure the length of cache is equal to the length of the requested sample

639 if cached_prices is not None and dater.to_string(end_date) in cached_prices.keys() and (

640 (asset_type == keys.keys['ASSETS']['EQUITY']

641 and (dater.business_days_between(start_date, end_date)) == len(cached_prices))

642 or

643 (asset_type == keys.keys['ASSETS']['CRYPTO']

644 and (dater.days_between(start_date, end_date)) == len(cached_prices))

645 ):

646 # TODO: debug the crypto out of date check.

647 return cached_prices

648

649 if cached_prices is not None:

650 logger.debug(

651 f'Cached {ticker} prices are out of date, passing request off to external service', 'get_daily_price_history')

652

653 prices = price_manager.get_prices(

654 ticker=ticker, start_date=start_date, end_date=end_date, asset_type=asset_type)

655

656 if cached_prices is not None:

657 new_prices = helper.complement_dict_keys(prices, cached_prices)[0]

658 else:

659 new_prices = prices

660

661 price_cache.save_rows(ticker, new_prices)

662

663 if not prices: 663 ↛ 664line 663 didn't jump to line 664, because the condition on line 663 was never true

664 raise errors.PriceError(

665 f'Prices could not be retrieved for {ticker}')

666

667 return prices

668

669

670def get_daily_price_latest(ticker: str, asset_type: Union[None, str] = None) -> float:

671 """

672 Returns the latest closing price for a given ticker symbol.

673

674 Parameters

675 ----------

676 1. **ticker**: ``str``

677 ticker symbol whose latest closing price is to be retrieved. \n \n

678 2. **asset_type**: str``

679 *Optional*. Asset type of the ticker whose history is to be retrieved. Will be calculated from the `ticker` symbol if not provided.

680 """

681 last_date = dater.this_date_or_last_trading_date()

682 prices = get_daily_price_history(

683 ticker=ticker, asset_type=asset_type, start_date=last_date, end_date=last_date)

684 first_element = helper.get_first_json_key(prices)

685 return prices[first_element][keys.keys['PRICES']['OPEN']]

686

687

688def get_daily_prices_latest(tickers: List[str], asset_types: Union[None, List[str]] = None):

689 if asset_types is None:

690 asset_types = [None for _ in tickers]

691 return {ticker: get_daily_price_latest(ticker, asset_types[i]) for i, ticker in enumerate(tickers)}

692

693

694def get_daily_fred_history(symbol: str, start_date: Union[date, None] = None, end_date: Union[date, None] = None) -> list:

695 """

696 Wrapper around external service request for financial statistics data constructed by the Federal Reserve Economic Data. Relies on an instance of `StatManager` configured by `settings.STAT_MANAGER` value, which in turn is configured by the `STAT_MANAGER` environment variable, to hydrate with data.

697

698 Parameters

699 ----------

700 1. **symbol**: ``str``

701 Symbol representing the statistic whose history is to be retrieved. List of allowable values can be found [here](https://www.quandl.com/data/FRED-Federal-Reserve-Economic-Data/documentation)

702 2. **start_date**: ``Union[date, None]``

703 *Optional*. Start date of price history. Defaults to None. If `start_date is None`, the calculation is made as if the `start_date` were set to 100 trading days ago. This excludes weekends and holidays.

704 3. **end_date**: ``Union[date, None]``

705 *Optional*. End date of price history. Defaults to None. If `end_date is None`, the calculation is made as if the `end_date` were set to today. This excludes weekends and holidays so that `end_date` is set to the last previous business date.

706

707 Returns

708 ------

709 ``list``: `{ 'date' (str) : value (str), 'date' (str): value (str), ... }`

710 Dictionary with date strings formatted `YYYY-MM-DD` as keys and the statistic on that date as the corresponding value.

711

712 Raises

713 ------

714 1. **scrilla.errors.PriceError**

715 If no sample prices can be retrieved, this error is thrown.

716

717 .. notes::

718 * Most financial statistics are not reported on weekends or holidays, so the `asset_type` for financial statistics is functionally equivalent to equities, at least as far as date calculations are concerned. The dates inputted into this function are validated as if they were labelled as equity `asset_types` for this reason.

719

720 """

721

722 start_date, end_date = errors.validate_dates(

723 start_date=start_date, end_date=end_date, asset_type=keys.keys['ASSETS']['EQUITY'])

724

725 stats = stat_manager.get_stats(

726 symbol=symbol, start_date=start_date, end_date=end_date)

727

728 if not stats:

729 raise errors.PriceError(

730 f'Prices could not be retrieved for {symbol}')

731

732 return stats

733

734

735def get_daily_fred_latest(symbol: str) -> float:

736 """

737 Returns the latest value for the inputted statistic symbol.

738

739 Parameters

740 ----------

741 1. **symbol**: str

742 Symbol representing the statistic whose value it to be retrieved.

743 """

744 stats_history = get_daily_fred_history(symbol=symbol)

745 first_element = helper.get_first_json_key(stats_history)

746 return stats_history[first_element]

747

748

749def get_daily_interest_history(maturity: str, start_date: Union[date, None] = None, end_date: Union[date, None] = None) -> list:

750 """

751 Wrapper around external service request for US Treasury Yield Curve data. Relies on an instance of `StatManager` configured by `settings.STAT_MANAGER` value, which in turn is configured by the `STAT_MANAGER` environment variable, to hydrate with data.

752

753 Before deferring to the `StatManager` and letting it call the external service, however, this function checks if response is in local cache. If the response is not in the cache, it will pass the request off to `StatManager` and then save the response in the cache so subsequent calls to the function can bypass the service request. Used to prevent excessive external HTTP requests and improve the performance of the application. Other parts of the program should interface with the external statistics data services through this function to utilize the cache functionality.

754

755 Parameters

756 ----------

757 1. **maturity** : ``str``

758 Maturity of the US Treasury for which the interest rate will be retrieved. List of allowable values can in `scrilla.stats.keys['SERVICES']['STATISTICS']['QUANDL']['MAP']['YIELD_CURVE']`

759 2. **start_date** : ``datetime.date``

760 *Optional*. Start date of price history. Defaults to None. If `start_date is None`, the calculation is made as if the `start_date` were set to 100 trading days ago. This excludes weekends and holidays.

761 3. **end_date** : ``datetime.date``

762 *Optional*. End date of price history. Defaults to None. If `end_date is None`, the calculation is made as if the `end_date` were set to today. This excludes weekends and holidays so that `end_date` is set to the last previous business date.

763

764 Returns

765 ------

766 ``dict`` : `{ 'date' : value , 'date': value , ... }`

767 Dictionary with date strings formatted `YYYY-MM-DD` as keys and the interest on that date as the corresponding value.

768

769 .. notes::

770 * Yield rates are not reported on weekends or holidays, so the `asset_type` for interest is functionally equivalent to equities, at least as far as date calculations are concerned. The dates inputted into this function are validated as if they were labelled as equity `asset_types` for this reason.

771 """

772 start_date, end_date = errors.validate_dates(

773 start_date=start_date, end_date=end_date, asset_type=keys.keys['ASSETS']['EQUITY'])

774

775 rates = interest_cache.filter(

776 maturity, start_date=start_date, end_date=end_date)

777

778 if rates is not None:

779 logger.debug(

780 f'Comparing cache-size({len(rates)}) = date-size({dater.business_days_between(start_date, end_date, True)})', 'get_daily_interest_history')

781

782 # TODO: this only works when stats are reported daily and that the latest date in the dataset is actually end_date.

783 if rates is not None and \

784 dater.to_string(end_date) in rates.keys() and \

785 dater.business_days_between(start_date, end_date) == len(rates):

786 return rates

787

788 logger.debug(

789 f'Cached {maturity} data is out of date, passing request to external service', 'get_daily_interest_history')

790 rates = stat_manager.get_interest_rates(

791 start_date=start_date, end_date=end_date)

792

793 interest_cache.save_rows(rates)

794

795 rates = stat_manager.format_for_maturity(maturity=maturity, results=rates)

796

797 return rates

798

799

800def get_daily_interest_latest(maturity: str) -> float:

801 """

802 Returns the latest interest rate for the inputted US Treasury maturity.

803

804 Parameters

805 ----------

806 1. **maturity**: ``str``

807 Maturity of the US Treasury security whose interest rate is to be retrieved. Allowable values accessible through `keys.keys['YIELD_CURVE']

808 """

809 end_date = dater.get_last_trading_date(True)

810 start_date = dater.decrement_date_by_business_days(end_date, 1, True)

811 interest_history = get_daily_interest_history(

812 maturity, start_date, end_date)

813 first_element = helper.get_first_json_key(interest_history)

814 return interest_history[first_element]

815

816

817def get_dividend_history(ticker: str) -> dict:

818 """

819 Wrapper around external service request for dividend payment data. Relies on an instance of `DivManager` configured by `settings.DIV_MANAGER` value, which in turn is configured by the `DIV_MANAGER` environment variable, to hydrate with data.

820

821 Note, since dividend payments do not occur every day (if only), dividend amounts do not get cached, as there is no nice way to determine on a given day whether or not a payment should have been made, and thus to determine whether or not the cache is out of date. In other words, you can't look at today's date and the date of the last payment in the cache and determine based solely on the dates whether or not the cache is outdated.

822

823 Parameters

824 ----------

825 1. **ticker** : ``str``

826 Ticker symbol of the equity whose dividend history is to be retrieved.

827

828 Returns

829 ------

830 ``list`` : `{ 'date' (str) : amount (str), 'date' (str): amount (str), ... }`

831 Dictionary with date strings formatted `YYYY-MM-DD` as keys and the dividend payment amount on that date as the corresponding value.

832 """

833 logger.debug(

834 f'Retrieving {ticker} dividends from service', 'get_dividend_history')

835 divs = div_manager.get_dividends(ticker=ticker)

836 return divs

837

838

839def get_risk_free_rate() -> float:

840 """

841 Returns the risk free rate, defined as the annualized yield on a specific US Treasury duration, as a decimal. The US Treasury yield used as a proxy for the risk free rate is defined in the `settings.py` file and is configured through the RISK_FREE environment variable.

842 """

843 return get_daily_interest_latest(maturity=settings.RISK_FREE_RATE)/100

844

845

846price_manager = PriceManager(settings.PRICE_MANAGER)

847stat_manager = StatManager(settings.STAT_MANAGER)

848div_manager = DividendManager(settings.DIV_MANAGER)

849price_cache = cache.PriceCache()

850interest_cache = cache.InterestCache()