Improvements

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include nsepy/resources *

nsepy/derivatives/__init__.py

@@ -2,24 +2,26 @@
 
 import datetime
 from datetime import date
-import re
+import dateutil.relativedelta
+import pandas
+from .. import live
+import pdb
+#import re
 
-from nsepy import urls
+# re_date = re.compile("([0-9]{2}\-[0-9]{2}\-[0-9]{4})")
+# idx_exp = {}
+# vix_exp = {}
+# stk_exp = {}
 
-re_date = re.compile("([0-9]{2}\-[0-9]{2}\-[0-9]{4})")
-idx_exp = {}
-vix_exp = {}
-stk_exp = {}
 
+# def add_dt(instru, dt):
+    # if not dt.year in instru:
+        # instru[dt.year] = {}
 
-def add_dt(instru, dt):
-    if not dt.year in instru:
-        instru[dt.year] = {}
+    # if not dt.month in instru[dt.year]:
+        # instru[dt.year][dt.month] = set()
 
-    if not dt.month in instru[dt.year]:
-        instru[dt.year][dt.month] = set()
-
-    instru[dt.year][dt.month].add(dt)
+    # instru[dt.year][dt.month].add(dt)
 
 
 class ExpiryDateError(Exception):
@@ -29,92 +31,136 @@ def __init__(self, message):
         super(ExpiryDateError, self).__init__(message)
 
 
-def build_dt_dict():
-    lines = urls.derivative_expiry_dates_url().text
-
-    for line in lines.split('\n'):
-        s = re_date.search(line)
-
-        if s:
-            dt = datetime.datetime.strptime(s.group(1), "%d-%m-%Y").date()
-            # Start Kludge
-            # The list on NSE portal for expiry date has a wrong entry for 20 Sep 2019
-            # Handle this oulier use case by ignoring this date and skpping it for processing
-            if dt == datetime.datetime(2019, 9, 20).date():
-                continue
-            # End Kludge
-            if line.find('indxExpryDt') > -1:
-                try:
-                    existing_date = try_to_get_expiry_date(
-                        dt.year, dt.month, index=True)
-                    if existing_date < dt:
-                        add_dt(idx_exp, dt)
-                except:
-                    add_dt(idx_exp, dt)
-
-            if line.find('stk') > -1:
-                try:
-                    existing_date = try_to_get_expiry_date(
-                        dt.year, dt.month, index=False, stock=False, vix=False)
-                    if existing_date < dt:
-                        add_dt(stk_exp, dt)
-                except:
-                    add_dt(stk_exp, dt)
-
-            if line.find('vix') > -1:
-                try:
-                    existing_date = try_to_get_expiry_date(
-                        dt.year, dt.month, index=False, stock=False, vix=True)
-                    if existing_date < dt:
-                        add_dt(vix_exp, dt)
-                except:
-                    add_dt(vix_exp, dt)
-
+# def build_dt_dict():
+    # lines = urls.derivative_expiry_dates_url().text
+
+    # for line in lines.split('\n'):
+        # s = re_date.search(line)
+
+        # if s:
+            # dt = datetime.datetime.strptime(s.group(1), "%d-%m-%Y").date()
+            # # Start Kludge
+            # # The list on NSE portal for expiry date has a wrong entries
+            # # Handle these oulier use cases by ignoring this date and skpping it for processing
+            # if dt == datetime.datetime(2019, 9, 20).date():
+                # continue
+            # if dt == datetime.datetime(2021, 11, 2).date():
+                # continue
+            # if dt == datetime.datetime(2021, 11, 9).date():
+                # continue
+            # if dt == datetime.datetime(2021, 11, 16).date():
+                # continue
+            # if dt == datetime.datetime(2021, 11, 23).date():
+                # continue
+            # if dt == datetime.datetime(2021, 11, 30).date():
+                # continue
+            # if dt == datetime.datetime(2021, 12, 7).date():
+                # continue
+            # if dt == datetime.datetime(2021, 12, 14).date():
+                # continue
+            # # End Kludge
+            # if line.find('indxExpryDt') > -1:
+                # try:
+                    # existing_date = try_to_get_expiry_date(
+                        # dt.year, dt.month, index=True)
+                    # if existing_date < dt:
+                        # add_dt(idx_exp, dt)
+                # except:
+                    # add_dt(idx_exp, dt)
+
+            # if line.find('stk') > -1:
+                # try:
+                    # existing_date = try_to_get_expiry_date(
+                        # dt.year, dt.month, index=False, stock=False, vix=False)
+                    # if existing_date < dt:
+                        # add_dt(stk_exp, dt)
+                # except:
+                    # add_dt(stk_exp, dt)
+
+            # if line.find('vix') > -1:
+                # try:
+                    # existing_date = try_to_get_expiry_date(
+                        # dt.year, dt.month, index=False, stock=False, vix=True)
+                    # if existing_date < dt:
+                        # add_dt(vix_exp, dt)
+                # except:
+                    # add_dt(vix_exp, dt)
+    # # Start Kludge
+    # # The list on NSE portal doesnt have entry for weekly expiry for 22 Oct 2020
+    # # Handle this oulier and add the entry explicity
+    # add_dt(idx_exp, datetime.datetime(2020, 10, 22).date())
 
 def is_valid_expiry(dt):
     # not a perfect logic :P
     if (dt.month != 2 and dt.day >= 23) or (dt.month == 2 and dt.day >= 21):
         return True
 
+def get_expiry_date(year, month, index=True, stock=False):
+
+    if not index ^ stock:
+       raise ValueError("index and stock params have to be XOR. Both can't be True of False at the same time.")
+
+    stexpiry = set()
+    monthstart = datetime.datetime(year, month, 1).date()
+    monthend =  (monthstart + dateutil.relativedelta.relativedelta(months=1)) - dateutil.relativedelta.relativedelta(days=1)
+
+    #Indices have weekly expiry every Thursday. If Thursday is a holiday then the previous working day
+    for d in pandas.date_range(start=monthstart , end=monthend, freq='W-THU').date:
+      #Make sure its not a holiday
+      stexpiry |= set([live.nearestworkingday(d)])
+
+    if index:
+      #Indices have weekly expiry
+      return stexpiry
+
+    elif stock:
+      #Indices have monthly expiry. Get last expirty of index
+      return set([max(stexpiry)])
+
 
-def try_to_get_expiry_date(year, month, index=True, stock=False, vix=False):
+    #for d in stprobableexpiry:
+    #   stexpiry |= live.nearestworkingday(d)
+
+    return stexpiry
+
+# def try_to_get_expiry_date(year, month, index=True, stock=False, vix=False):
 
-    try:
-        if vix and vix_exp:
-            return vix_exp[year][month]
+    # try:
+        # if vix and vix_exp:
+            # return vix_exp[year][month]
 
-        if stock and stk_exp:
-            return stk_exp[year][month]
+        # if stock and stk_exp:
+            # return stk_exp[year][month]
 
-        if index and idx_exp:
-            return idx_exp[year][month]
+        # if index and idx_exp:
+            # return idx_exp[year][month]
 
-        raise Exception
-    except:
+        # raise Exception
+    # except:
 
-        if index:
-            name = 'index derivatives'
-        if stock:
-            name = 'stock derivatives'
-        else:
-            name = 'vix derivatives'
-        raise ExpiryDateError(
-            'No expiry date found in the month of {}-{} for {}'.format(year, month, name))
+        # if index:
+            # name = 'index derivatives'
+        # if stock:
+            # name = 'stock derivatives'
+        # else:
+            # name = 'vix derivatives'
+        # raise ExpiryDateError(
+            # 'No expiry date found in the month of {}-{} for {}'.format(year, month, name))
 
 
-def get_expiry_date(year, month, index=True, stock=False, vix=False, recursion=0):
+# def get_expiry_date(year, month, index=True, stock=False, vix=False, recursion=0):
 
-    try:
-        return try_to_get_expiry_date(year, month, index, stock, vix)
-    except:
+    # try:
+        # return try_to_get_expiry_date(year, month, index, stock, vix)
+    # except:
 
-        if recursion > 1:
-            raise
+        # if recursion > 1:
+            # raise
 
-        else:
-            pass
+        # else:
+            # pass
 
-    #print("building dictionary")
+    # #print("building dictionary")
 
-    build_dt_dict()
-    return get_expiry_date(year, month, index, stock, vix, recursion=recursion+1)
+    # build_dt_dict()
+    # return get_expiry_date(year, month, index, stock, vix, recursion=recursion+1)

nsepy/history.py

@@ -324,7 +324,10 @@ def get_price_list(dt, series='EQ'):
     txt = unzip_str(res.content)
     fp = six.StringIO(txt)
     df = pd.read_csv(fp)
-    del df['Unnamed: 13']
+
+    #Delete column Unnamed: 13 if exits
+    if 'Unnamed: 13' in df.columns :
+      df = df.drop(columns=['Unnamed: 13'])
     return df[df['SERIES'] == series]
 
 

nsepy/live.py

@@ -10,7 +10,9 @@
 import ast
 import json
 import io
+import pandas as pd
 from bs4 import BeautifulSoup
+import pkg_resources
 from nsepy.liveurls import quote_eq_url, quote_derivative_url, option_chain_url, futures_chain_url, holiday_list_url
 
 
@@ -109,7 +111,7 @@ def get_futures_chain_table(symbol):
     sptable = spdiv.find("table")
     tp = ParseTables(soup=sptable, schema=FUTURES_SCHEMA,
                      headers=FUTURES_HEADERS, index=FUTURES_INDEX)
-    return tp.get_df()
+    return tp.get_df().sort_index()
 
 
 def get_holidays_list(fromDate,
@@ -127,16 +129,21 @@ def get_holidays_list(fromDate,
     if fromDate > toDate:
         raise ValueError('Please check start and end dates')
 
-    holidayscrape = holiday_list_url(fromDate.strftime(
-        "%d-%m-%Y"), toDate.strftime("%d-%m-%Y"))
-    html_soup = BeautifulSoup(holidayscrape.text, 'lxml')
-    sptable = html_soup.find("table")
-    tp = ParseTables(soup=sptable,
-                     schema=[str, StrDate.default_format(
-                         format="%d-%b-%Y"), str, str],
-                     headers=["Market Segment", "Date", "Day", "Description"], index="Date")
-    dfret = tp.get_df()
-    dfret = dfret.drop(["Market Segment"], axis=1)
+    # holidayscrape = holiday_list_url(fromDate.strftime(
+        # "%d-%m-%Y"), toDate.strftime("%d-%m-%Y"))
+    # html_soup = BeautifulSoup(holidayscrape.text, 'lxml')
+    # sptable = html_soup.find("table")
+    # tp = ParseTables(soup=sptable,
+                     # schema=[str, StrDate.default_format(
+                         # format="%d-%b-%Y"), str, str],
+                     # headers=["Market Segment", "Date", "Day", "Description"], index="Date")
+    # dfret = tp.get_df()
+    strmholiday = pkg_resources.resource_stream(__name__, 'resources/holiday.csv')
+    dfdata = pd.read_csv(strmholiday,parse_dates=['Date'])
+    dfret = dfdata.drop(["Market Segment"], axis=1)
+    dfret['Date'] = dfret['Date'].dt.date
+    dfret = dfret.set_index('Date')
+    dfret = dfret.loc[fromDate:toDate]
     return dfret
 
 
@@ -187,6 +194,18 @@ def previousworkingday(dt):
         if isworkingday(dttmp):
             return dttmp
 
+def nearestworkingday(dt):
+    """This is the function to get nearest working day.
+        Args:
+            dt (datetime.date): Date to Check
+        Returns:
+            dt (datetime.date): Nearest working day on or before the given date
+    """
+    if isworkingday (dt):
+      return dt
+    else:
+      return previousworkingday(dt)
+
 
 def getworkingdays(dtfrom, dtto):
     # pdb.set_trace()

nsepy/liveurls.py

@@ -15,8 +15,10 @@
            'Accept-Language': 'en-GB,en-US;q=0.8,en;q=0.6',
            'Connection': 'keep-alive',
            'Host': 'www1.nseindia.com',
-           'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
-           'X-Requested-With': 'XMLHttpRequest'}
+           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
+           #'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+           'X-Requested-With': 'XMLHttpRequest',
+           }
 
 
 """