Commits for 10-15-2023

2024_blue_chip_ratio.py

@@ -0,0 +1,27 @@
+import pandas as pd
+import os
+import re
+import numpy as np
+
+#get current working directory
+path = os.getcwd()
+
+#concatenate json files
+df1 = pd.read_json(os.path.join('data', '2024_data.json'))
+df2 = pd.read_json(os.path.join('data', '2023_data.json'))
+df3 = pd.read_json(os.path.join('data', '2022_data.json'))
+df4 = pd.read_json(os.path.join('data', '2021_data.json'))
+
+frames = [df1, df2, df3, df4]
+result1 = pd.concat(frames)
+result2 = result1.groupby('team').aggregate(['sum'])
+result2.columns = ['total_commits_sum', 'five_star_sum', 'four_star_sum', 'three_star_sum']
+result2['blue_chip'] = result2['five_star_sum'] + result2['four_star_sum']
+result2['blue_chip_ratio'] = result2['blue_chip']/result2['total_commits_sum']
+result2 = result2.sort_values(by='blue_chip_ratio', ascending=False)
+result2.columns = ['Total Commits', '5-Star total', '4-Star total', '3-Star total', 'Blue Chips Total', '2024 Blue Chip Ratio']
+result = result2['2024 Blue Chip Ratio']
+print(result)
+#path=r'C:\Users\carandangc\Desktop\testscrape\'
+#path = os.getcwd()
+#result.to_csv(os.path.join('data', '2021_blue_chip_ratio.csv'))

2024_scraper.py

@@ -0,0 +1,78 @@
+from selenium import webdriver
+from bs4 import BeautifulSoup
+import re
+import pandas as pd
+import numpy as np
+import time
+from tabulate import tabulate
+import os
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait 
+from selenium.webdriver.common.action_chains import ActionChains
+
+#launch url
+url = "https://247sports.com/season/2024-football/compositeteamrankings/"
+
+# create a new Firefox session
+driver = webdriver.Firefox(service_log_path = os.devnull)
+driver.implicitly_wait(30)
+driver.get(url)
+
+#scroll down and click to load more
+for n in range(10):
+    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+    driver.implicitly_wait(120)
+    try:
+    	python_button = driver.find_element_by_partial_link_text('Load')
+    except:
+    	break
+    try:
+    	python_button.click() #click link
+    except:
+    	break
+
+datalist = [] #empty list
+
+#Selenium hands the page source to Beautiful Soup
+soup_level1=BeautifulSoup(driver.page_source, "lxml")
+
+table = soup_level1.find_all('li', attrs={'class':'rankings-page__list-item'})
+
+i = 0
+for i in range(len(table)):
+    try:
+        team = table[i].div.a.text.strip()
+        if team == '':
+            team = table[i].find('div', attrs={'class','team'}).text.strip()
+    except:
+        team = np.nan
+    total_commits = table[i].find('div', attrs={'class','total'}).text.split()[0]
+    five_star = table[i].ul.text.split()[1]
+    four_star = table[i].ul.text.split()[3]
+    try:
+        three_star = table[i].ul.text.split()[5]
+    except:
+        three_star = np.nan
+    datalist.append((team, total_commits, five_star, four_star, three_star))
+
+#end the Selenium browser session
+driver.quit()
+
+#combine all pandas dataframes in the list into one big dataframe
+result = pd.DataFrame(datalist, columns=['team', 'total_commits', 'five_star', 'four_star', 'three_star'])
+
+#convert the pandas dataframe to JSON
+json_records = result.to_json(orient='records')
+
+#pretty print to CLI with tabulate
+#converts to an ascii table
+print(tabulate(result, headers=['team', 'total_commits', 'five_star', 'four_star', 'three_star'],tablefmt='psql'))
+
+#get current working directory
+path = os.getcwd()
+
+#open, write, and close the file
+f = open(os.path.join('data', '2024_data.json'), "w")
+f.write(json_records)
+f.close()

data/2024_data.json

@@ -0,0 +1 @@
+[{"team":"Georgia","total_commits":"26","five_star":"4","four_star":"17","three_star":"5"},{"team":"Ohio State","total_commits":"23","five_star":"5","four_star":"14","three_star":"3"},{"team":"Florida","total_commits":"21","five_star":"2","four_star":"14","three_star":"5"},{"team":"Texas A&M","total_commits":"22","five_star":"2","four_star":"17","three_star":"3"},{"team":"Alabama","total_commits":"20","five_star":"2","four_star":"12","three_star":"6"},{"team":"Florida State","total_commits":"21","five_star":"2","four_star":"12","three_star":"7"},{"team":"Oklahoma","total_commits":"25","five_star":"2","four_star":"13","three_star":"10"},{"team":"Notre Dame","total_commits":"23","five_star":"1","four_star":"15","three_star":"7"},{"team":"LSU","total_commits":"25","five_star":"0","four_star":"19","three_star":"6"},{"team":"Oregon","total_commits":"24","five_star":"0","four_star":"19","three_star":"5"},{"team":"Tennessee","total_commits":"20","five_star":"2","four_star":"13","three_star":"5"},{"team":"Penn State","total_commits":"24","five_star":"0","four_star":"16","three_star":"8"},{"team":"Miami","total_commits":"23","five_star":"0","four_star":"9","three_star":"14"},{"team":"Michigan","total_commits":"25","five_star":"0","four_star":"17","three_star":"8"},{"team":"Texas","total_commits":"18","five_star":"2","four_star":"13","three_star":"3"},{"team":"Clemson","total_commits":"17","five_star":"2","four_star":"10","three_star":"5"},{"team":"Auburn","total_commits":"17","five_star":"2","four_star":"12","three_star":"3"},{"team":"USC","total_commits":"15","five_star":"0","four_star":"10","three_star":"5"},{"team":"South Carolina","total_commits":"15","five_star":"1","four_star":"8","three_star":"5"},{"team":"Arkansas","total_commits":"19","five_star":"0","four_star":"8","three_star":"11"},{"team":"Ole Miss","total_commits":"23","five_star":"0","four_star":"7","three_star":"14"},{"team":"Kentucky","total_commits":"21","five_star":"0","four_star":"7","three_star":"13"},{"team":"Texas Tech","total_commits":"20","five_star":"1","four_star":"4","three_star":"14"},{"team":"Stanford","total_commits":"27","five_star":"0","four_star":"6","three_star":"21"},{"team":"Wisconsin","total_commits":"21","five_star":"0","four_star":"8","three_star":"13"},{"team":"Nebraska","total_commits":"26","five_star":"0","four_star":"8","three_star":"18"},{"team":"North Carolina","total_commits":"26","five_star":"0","four_star":"8","three_star":"18"},{"team":"Purdue","total_commits":"23","five_star":"0","four_star":"4","three_star":"18"},{"team":"UCF","total_commits":"18","five_star":"0","four_star":"9","three_star":"9"},{"team":"Georgia Tech","total_commits":"25","five_star":"0","four_star":"3","three_star":"22"},{"team":"NC State","total_commits":"20","five_star":"0","four_star":"3","three_star":"17"},{"team":"Minnesota","total_commits":"23","five_star":"0","four_star":"2","three_star":"21"},{"team":"Iowa","total_commits":"20","five_star":"0","four_star":"3","three_star":"17"},{"team":"Rutgers","total_commits":"23","five_star":"0","four_star":"2","three_star":"21"},{"team":"Pittsburgh","total_commits":"21","five_star":"0","four_star":"1","three_star":"20"},{"team":"Duke","total_commits":"20","five_star":"0","four_star":"3","three_star":"17"},{"team":"TCU","total_commits":"17","five_star":"0","four_star":"5","three_star":"11"},{"team":"Maryland","total_commits":"21","five_star":"0","four_star":"2","three_star":"19"},{"team":"Missouri","total_commits":"15","five_star":"1","four_star":"3","three_star":"11"},{"team":"Vanderbilt","total_commits":"20","five_star":"0","four_star":"2","three_star":"18"},{"team":"Virginia Tech","total_commits":"16","five_star":"0","four_star":"3","three_star":"13"},{"team":"Mississippi State","total_commits":"15","five_star":"0","four_star":"4","three_star":"11"},{"team":"Cincinnati","total_commits":"21","five_star":"0","four_star":"1","three_star":"20"},{"team":"Arizona","total_commits":"21","five_star":"0","four_star":"2","three_star":"19"},{"team":"Washington","total_commits":"14","five_star":"0","four_star":"5","three_star":"9"},{"team":"West Virginia","total_commits":"20","five_star":"0","four_star":"0","three_star":"20"},{"team":"Wake Forest","total_commits":"19","five_star":"0","four_star":"0","three_star":"19"},{"team":"Illinois","total_commits":"17","five_star":"0","four_star":"1","three_star":"16"},{"team":"Kansas","total_commits":"15","five_star":"0","four_star":"3","three_star":"12"},{"team":"Arizona State","total_commits":"17","five_star":"0","four_star":"0","three_star":"15"},{"team":"Indiana","total_commits":"19","five_star":"0","four_star":"0","three_star":"17"},{"team":"Syracuse","total_commits":"18","five_star":"0","four_star":"1","three_star":"16"},{"team":"Iowa State","total_commits":"18","five_star":"0","four_star":"0","three_star":"18"},{"team":"Louisville","total_commits":"14","five_star":"0","four_star":"3","three_star":"11"},{"team":"UCLA","total_commits":"12","five_star":"0","four_star":"3","three_star":"9"},{"team":"Washington State","total_commits":"18","five_star":"0","four_star":"1","three_star":"15"},{"team":"USF","total_commits":"17","five_star":"0","four_star":"0","three_star":"17"},{"team":"East Carolina","total_commits":"20","five_star":"0","four_star":"0","three_star":"18"},{"team":"UNLV","total_commits":"22","five_star":"0","four_star":"0","three_star":"20"},{"team":"Baylor","total_commits":"14","five_star":"0","four_star":"0","three_star":"13"},{"team":"Oklahoma State","total_commits":"13","five_star":"0","four_star":"0","three_star":"13"},{"team":"Oregon State","total_commits":"14","five_star":"0","four_star":"0","three_star":"14"},{"team":"California","total_commits":"12","five_star":"0","four_star":"0","three_star":"12"},{"team":"Michigan State","total_commits":"10","five_star":"0","four_star":"3","three_star":"7"},{"team":"Colorado State","total_commits":"17","five_star":"0","four_star":"1","three_star":"16"},{"team":"Utah","total_commits":"10","five_star":"0","four_star":"3","three_star":"7"},{"team":"Virginia","total_commits":"13","five_star":"0","four_star":"0","three_star":"12"},{"team":"Colorado","total_commits":"9","five_star":"0","four_star":"3","three_star":"6"},{"team":"Tulane","total_commits":"12","five_star":"0","four_star":"0","three_star":"12"},{"team":"Brigham Young","total_commits":"12","five_star":"0","four_star":"1","three_star":"11"},{"team":"Appalachian State","total_commits":"20","five_star":"0","four_star":"0","three_star":"19"},{"team":"Southern Miss","total_commits":"17","five_star":"0","four_star":"0","three_star":"15"},{"team":"Boise State","total_commits":"14","five_star":"0","four_star":"1","three_star":"11"},{"team":"Miami (OH)","total_commits":"17","five_star":"0","four_star":"0","three_star":"16"},{"team":"Florida Atlantic","total_commits":"15","five_star":"0","four_star":"0","three_star":"14"},{"team":"Coastal Carolina","total_commits":"15","five_star":"0","four_star":"0","three_star":"13"},{"team":"North Texas","total_commits":"17","five_star":"0","four_star":"0","three_star":"12"},{"team":"Toledo","total_commits":"12","five_star":"0","four_star":"0","three_star":"12"},{"team":"SMU","total_commits":"9","five_star":"0","four_star":"1","three_star":"8"},{"team":"Boston College","total_commits":"10","five_star":"0","four_star":"0","three_star":"10"},{"team":"Central Michigan","total_commits":"17","five_star":"0","four_star":"0","three_star":"14"},{"team":"James Madison","total_commits":"15","five_star":"0","four_star":"0","three_star":"12"},{"team":"Memphis","total_commits":"13","five_star":"0","four_star":"0","three_star":"10"},{"team":"Kent State","total_commits":"16","five_star":"0","four_star":"0","three_star":"13"},{"team":"Connecticut","total_commits":"12","five_star":"0","four_star":"0","three_star":"12"},{"team":"Northwestern","total_commits":"10","five_star":"0","four_star":"0","three_star":"9"},{"team":"Northern Illinois","total_commits":"16","five_star":"0","four_star":"0","three_star":"12"},{"team":"Tulsa","total_commits":"12","five_star":"0","four_star":"0","three_star":"9"},{"team":"Charlotte","total_commits":"16","five_star":"0","four_star":"0","three_star":"10"},{"team":"Marshall","total_commits":"15","five_star":"0","four_star":"0","three_star":"10"},{"team":"San Diego State","total_commits":"9","five_star":"0","four_star":"0","three_star":"9"},{"team":"Texas State","total_commits":"15","five_star":"0","four_star":"0","three_star":"9"},{"team":"Houston","total_commits":"8","five_star":"0","four_star":"1","three_star":"7"},{"team":"Eastern Michigan","total_commits":"16","five_star":"0","four_star":"0","three_star":"10"},{"team":"Fresno State","total_commits":"10","five_star":"0","four_star":"0","three_star":"10"},{"team":"South Alabama","total_commits":"15","five_star":"0","four_star":"0","three_star":"9"},{"team":"Kansas State","total_commits":"7","five_star":"0","four_star":"0","three_star":"7"},{"team":"Liberty","total_commits":"14","five_star":"0","four_star":"0","three_star":"9"},{"team":"San Jose State","total_commits":"10","five_star":"0","four_star":"0","three_star":"9"},{"team":"Akron","total_commits":"10","five_star":"0","four_star":"0","three_star":"9"}]