mirror of
https://github.com/peter-tanner/Blackboard-marks.git
synced 2024-11-30 11:40:16 +08:00
Download feedback in comments, and format code
This commit is contained in:
parent
e3ed2765d6
commit
234cd31bf4
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -4,3 +4,5 @@ __pycache__
|
|||
chromedriver*
|
||||
test*
|
||||
.vscode/
|
||||
*.7z
|
||||
*.tar
|
||||
70
main.py
70
main.py
|
|
@ -31,6 +31,7 @@ try:
|
|||
except:
|
||||
def get_etc(*args): return False
|
||||
|
||||
|
||||
# stupid bug
|
||||
def click_the_fing_button(driver, button):
|
||||
try:
|
||||
|
|
@ -38,17 +39,22 @@ def click_the_fing_button(driver,button):
|
|||
ActionChains(driver).click(button).perform()
|
||||
WebDriverWait(driver, 2).until(EC.number_of_windows_to_be(2))
|
||||
except:
|
||||
driver.set_window_size(1024, 768) # hack to wake selenium up when it doesnt want to click the button!
|
||||
# hack to wake selenium up when it doesnt want to click the button!
|
||||
driver.set_window_size(1024, 768)
|
||||
click_the_fing_button(driver, button)
|
||||
driver.maximize_window()
|
||||
|
||||
# You can probably replace this with a recursive method like in blackboard scraper but tbh i just want to get this script done so i can stop working for once.
|
||||
|
||||
|
||||
def scrape_further(driver, path, session):
|
||||
# attempts for bb-held tests
|
||||
attempts = driver.find_elements(By.XPATH, "//a[starts-with(@href, '/webapps/assessment')]")
|
||||
attempts = driver.find_elements(
|
||||
By.XPATH, "//a[starts-with(@href, '/webapps/assessment')]")
|
||||
attempts = [x.get_attribute('href') for x in attempts]
|
||||
for i, attempt in enumerate(attempts):
|
||||
name = "attempt_"+str(i)+"_["+parse_qs(urlparse(attempt).query)['attempt_id'][0]+"]"
|
||||
name = "attempt_" + \
|
||||
str(i)+"_["+parse_qs(urlparse(attempt).query)['attempt_id'][0]+"]"
|
||||
attempt = re.sub("^"+BASE_URL, "", attempt)
|
||||
driver.execute_script("window.open('"+BASE_URL+attempt+"')")
|
||||
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(3))
|
||||
|
|
@ -59,9 +65,18 @@ def scrape_further(driver,path,session):
|
|||
driver.close()
|
||||
driver.switch_to.window(driver.window_handles[1])
|
||||
|
||||
# submission file for assignment
|
||||
# Comments may contain feedback links
|
||||
request_stack = RequestStack(session)
|
||||
attempts = driver.find_elements(By.XPATH, "//a[starts-with(@href, '/webapps/assignment/download')]")
|
||||
etc_files = driver.find_elements(
|
||||
By.XPATH, "//a[contains(@href, '/bbcswebdav')]")
|
||||
etc_files = [x.get_attribute('href') for x in etc_files]
|
||||
for i, item in enumerate(etc_files):
|
||||
if (not item is None) and ("bbcswebdav" in item):
|
||||
request_stack.add_file(item, path)
|
||||
|
||||
# submission file for assignment
|
||||
attempts = driver.find_elements(
|
||||
By.XPATH, "//a[starts-with(@href, '/webapps/assignment/download')]")
|
||||
attempts = [x.get_attribute('href') for x in attempts]
|
||||
for i, attempt in enumerate(attempts):
|
||||
request_stack.add_file(attempt, path)
|
||||
|
|
@ -71,13 +86,15 @@ def scrape_further(driver,path,session):
|
|||
# download button causes a tab to appear quickly, download, then disappear
|
||||
# need to capture the url to get the metadata and dl to the correct location
|
||||
# cant be arsed to figure out how the pspdfkit js that executes this download works.
|
||||
SwitchToIFrame(driver, (By.XPATH, "//iframe[@class='docviewer_iframe_embed']"))
|
||||
SwitchToIFrame(
|
||||
driver, (By.XPATH, "//iframe[@class='docviewer_iframe_embed']"))
|
||||
SwitchToIFrame(driver, (By.XPATH, "//iframe[@title='PSPDFKit']"))
|
||||
get_feedback = True
|
||||
except:
|
||||
print("No feedback to download")
|
||||
if get_feedback:
|
||||
dl_button = WaitClickable(driver,(By.XPATH, "//button[contains(@class,'PSPDFKit-Toolbar-Button PSPDFKit-Tool-Button')][@title='Download']"))
|
||||
dl_button = WaitClickable(
|
||||
driver, (By.XPATH, "//button[contains(@class,'PSPDFKit-Toolbar-Button PSPDFKit-Tool-Button')][@title='Download']"))
|
||||
dl_button.click()
|
||||
download_file(path)
|
||||
request_stack.download_all()
|
||||
|
|
@ -86,7 +103,8 @@ def scrape_further(driver,path,session):
|
|||
|
||||
parser = argparse.ArgumentParser(description='Automated microsoft SSO login.')
|
||||
# parser.add_argument("-p", "--password", help="Automatically use provided password", default="")
|
||||
parser.add_argument("-u", "--username", help="Automatically use provided userID", default="")
|
||||
parser.add_argument("-u", "--username",
|
||||
help="Automatically use provided userID", default="")
|
||||
|
||||
path = ['grades']
|
||||
args = parser.parse_args()
|
||||
|
|
@ -104,6 +122,8 @@ prefs = {
|
|||
"directory_upgrade": True
|
||||
}
|
||||
OPTIONS = Options()
|
||||
OPTIONS.add_argument('--no-sandbox')
|
||||
OPTIONS.add_argument('--disable-dev-shm-usage')
|
||||
OPTIONS.add_experimental_option("prefs", prefs)
|
||||
# OPTIONS.add_argument("--headless")
|
||||
driver = webdriver.Chrome(
|
||||
|
|
@ -125,9 +145,11 @@ try:
|
|||
except:
|
||||
print("no tos warning - skipped")
|
||||
|
||||
driver.get(BASE_URL+"/webapps/streamViewer/streamViewer?cmd=view&streamName=mygrades")
|
||||
driver.get(
|
||||
BASE_URL+"/webapps/streamViewer/streamViewer?cmd=view&streamName=mygrades")
|
||||
save_html(sep.join(path), 'entrypoint', driver.page_source)
|
||||
|
||||
WaitClickable(driver, (By.ID, "left_stream_mygrades"))
|
||||
# get courseIDs
|
||||
courses = driver.find_element(By.ID, "left_stream_mygrades")\
|
||||
.find_elements(By.XPATH, "//div[@role='tab']")
|
||||
|
|
@ -137,8 +159,10 @@ for i, course_results in enumerate(courses):
|
|||
course_results = courses[i]
|
||||
ActionChains(driver).move_to_element(course_results).perform()
|
||||
course_url = course_results.get_attribute("bb:rhs")
|
||||
course_name = course_results.find_elements(By.XPATH, "//span[@class='stream_area_name']")[i].text
|
||||
course_name += " ["+parse_qs(urlparse(course_url).query)['course_id'][0]+"]"
|
||||
course_name = course_results.find_elements(
|
||||
By.XPATH, "//span[@class='stream_area_name']")[i].text
|
||||
course_name += " [" + \
|
||||
parse_qs(urlparse(course_url).query)['course_id'][0]+"]"
|
||||
course_details.append({
|
||||
'name': course_name,
|
||||
'url': course_url
|
||||
|
|
@ -167,16 +191,19 @@ for i, course in enumerate(course_details):
|
|||
assignment_name = None
|
||||
information_link = False
|
||||
try:
|
||||
block = assignment.find_element(By.XPATH, "./div[@class='cell gradable']/a[@onclick]")
|
||||
block = assignment.find_element(
|
||||
By.XPATH, "./div[@class='cell gradable']/a[@onclick]")
|
||||
information_link = True
|
||||
except:
|
||||
block = assignment.find_element(By.XPATH, "./div[@class='cell gradable']")
|
||||
block = assignment.find_element(
|
||||
By.XPATH, "./div[@class='cell gradable']")
|
||||
assignment_name = get_assignment_name(driver, block)
|
||||
path.append(assignment_name)
|
||||
# download information if it exists.
|
||||
if information_link:
|
||||
try:
|
||||
ActionChains(driver).move_to_element(block).click(block).perform()
|
||||
ActionChains(driver).move_to_element(
|
||||
block).click(block).perform()
|
||||
print("Switched "+assignment_name)
|
||||
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
|
||||
driver.switch_to.window(driver.window_handles[1])
|
||||
|
|
@ -191,14 +218,17 @@ for i, course in enumerate(course_details):
|
|||
action = button.get_attribute("onclick")
|
||||
if action != None and "showInLightBox" not in action:
|
||||
click_the_fing_button(driver, button)
|
||||
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
|
||||
driver.execute_script(
|
||||
"window.scrollTo(0, document.body.scrollHeight)")
|
||||
driver.switch_to.window(driver.window_handles[1])
|
||||
WaitDiv(driver, (By.CLASS_NAME, "rubricControlContainer"))
|
||||
save_html(sep.join(path), "rubric", driver.page_source)
|
||||
driver.find_element(By.XPATH, "//li[@id='listViewTab']/a").click()
|
||||
driver.find_element(
|
||||
By.XPATH, "//li[@id='listViewTab']/a").click()
|
||||
WaitDiv(driver, (By.CLASS_NAME, "rubricGradingList"))
|
||||
save_html(sep.join(path), "list", driver.page_source)
|
||||
detailed_buttons = driver.find_elements(By.XPATH, "//div[@class='u_controlsWrapper']/input")
|
||||
detailed_buttons = driver.find_elements(
|
||||
By.XPATH, "//div[@class='u_controlsWrapper']/input")
|
||||
detailed_buttons[1].click()
|
||||
detailed_buttons[0].click()
|
||||
save_html(sep.join(path), "list_detailed", driver.page_source)
|
||||
|
|
@ -209,8 +239,10 @@ for i, course in enumerate(course_details):
|
|||
WaitClickable(driver, (By.XPATH, "//a[@value='S']")).click()
|
||||
save_html(sep.join(path), "submitted", driver.page_source)
|
||||
try:
|
||||
WaitClickable(driver,(By.XPATH,"//div[@id='submissionReceipts']//a")).click()
|
||||
WaitClickable(driver,(By.XPATH,"//div[@id='listContainer_itemcount']//a[@class='pagelink']")).click()
|
||||
WaitClickable(
|
||||
driver, (By.XPATH, "//div[@id='submissionReceipts']//a")).click()
|
||||
WaitClickable(
|
||||
driver, (By.XPATH, "//div[@id='listContainer_itemcount']//a[@class='pagelink']")).click()
|
||||
except:
|
||||
print('No items?')
|
||||
save_html(sep.join(path), "receipts", driver.page_source)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# https://stackoverflow.com/a/49375740
|
||||
import os, sys
|
||||
import os
|
||||
import sys
|
||||
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
|
@ -6,6 +6,7 @@ import shutil
|
|||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def convert_filename(name, hash):
|
||||
_name = name.split('.')
|
||||
if len(_name) > 1:
|
||||
|
|
@ -14,6 +15,7 @@ def convert_filename(name, hash):
|
|||
_name[0] += ("["+hash+"]")
|
||||
return '.'.join(_name)
|
||||
|
||||
|
||||
class RequestStack:
|
||||
def __init__(self, token):
|
||||
self.request_stack = []
|
||||
|
|
@ -28,6 +30,7 @@ class RequestStack:
|
|||
print(f"\tDownloading {file.url}")
|
||||
file.download(self.token)
|
||||
|
||||
|
||||
class Asset:
|
||||
def __init__(self, url, path):
|
||||
self.path = Path(path)
|
||||
|
|
@ -37,7 +40,8 @@ class Asset:
|
|||
super().__init__()
|
||||
|
||||
def download(self, session):
|
||||
response = session.get(BASE_URL+self.url, stream=True, allow_redirects=False)
|
||||
response = session.get(
|
||||
BASE_URL+self.url, stream=True, allow_redirects=False)
|
||||
headers = response.headers
|
||||
if response.status_code == 302 and len(headers['location']) > 0:
|
||||
Asset(headers['location'], self.path).download(session)
|
||||
|
|
@ -45,13 +49,16 @@ class Asset:
|
|||
elif response.status_code != 200:
|
||||
print("[!] Error "+str(response.status_code))
|
||||
return response.status_code
|
||||
headers = { x:re.sub(r'^"*|"*?$', '', headers.get(x)) for x in headers } # ewww regex
|
||||
headers = {x: re.sub(r'^"*|"*?$', '', headers.get(x))
|
||||
for x in headers} # ewww regex
|
||||
if 'Content-Disposition' in headers.keys():
|
||||
self.original_filename = re.findall('filename="(.+)"', headers['Content-Disposition'])[0]
|
||||
self.original_filename = re.findall(
|
||||
'filename="(.+)"', headers['Content-Disposition'])[0]
|
||||
else:
|
||||
self.original_filename = re.sub(".*/", "", self.url)
|
||||
self.etag_hash = hashlib.md5(headers['ETag'].encode()).hexdigest()
|
||||
self.filename = convert_filename(self.original_filename, self.etag_hash[0:6])
|
||||
self.filename = convert_filename(
|
||||
self.original_filename, self.etag_hash[0:6])
|
||||
|
||||
with open(self.path.joinpath(self.filename), 'wb') as f:
|
||||
shutil.copyfileobj(response.raw, f)
|
||||
|
|
@ -62,7 +69,8 @@ class Asset:
|
|||
["original_filename", self.original_filename],
|
||||
["readable_filename", self.filename],
|
||||
["url", self.url],
|
||||
["pathhash", hashlib.md5(self.url.encode()).hexdigest()],
|
||||
["pathhash", hashlib.md5(
|
||||
self.url.encode()).hexdigest()],
|
||||
["etag", headers['ETag']],
|
||||
["etaghash", self.etag_hash],
|
||||
["last-modified", headers["Last-Modified"]],
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from constants.constants import BASE_URL
|
|||
import re
|
||||
import json
|
||||
|
||||
|
||||
def login(args, driver):
|
||||
driver.get(BASE_URL)
|
||||
USERNAME = args.username
|
||||
|
|
@ -28,7 +29,8 @@ def login(args, driver):
|
|||
WaitClickable(driver, Selectors.BUTTON_NEXT).click()
|
||||
print('Entered password.')
|
||||
except:
|
||||
print(WebDriverWait(driver, 1).until(EC.visibility_of_element_located(Selectors.DIV_USERERROR)).text)
|
||||
print(WebDriverWait(driver, 1).until(
|
||||
EC.visibility_of_element_located(Selectors.DIV_USERERROR)).text)
|
||||
driver.quit()
|
||||
exit(2)
|
||||
|
||||
|
|
@ -36,7 +38,8 @@ def login(args, driver):
|
|||
# WaitClickable(driver,BUTTON_NEXT).click() #IF you want to remember credentials, switch these comments
|
||||
|
||||
cookie = driver.get_cookies()
|
||||
if not cookie == None: return cookie
|
||||
if not cookie == None:
|
||||
return cookie
|
||||
|
||||
print('Could not get auth cookie - Invalid ID or password?', file=sys.stderr)
|
||||
driver.quit()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from selenium.webdriver.common.by import By
|
||||
|
||||
|
||||
class Selectors:
|
||||
# Microsoft login
|
||||
BOX_USERNAME = (By.ID, "i0116")
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ import os
|
|||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
|
||||
def friendly_filename(name):
|
||||
name = friendly_dirname(name)
|
||||
return re.sub("[\\\/]", '', name)
|
||||
|
||||
|
||||
def friendly_dirname(name):
|
||||
# .gsub(/[^\w\s_-]+/, '')
|
||||
# .gsub(/\s+/, '_')
|
||||
|
|
@ -26,6 +28,7 @@ def get_assignment_name(driver,block):
|
|||
print("Assesment: "+s)
|
||||
return s
|
||||
|
||||
|
||||
def save_html(dir, filename, page_source):
|
||||
dir = pathlib.Path(friendly_dirname(dir))
|
||||
dir.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -33,15 +36,12 @@ def save_html(dir,filename,page_source):
|
|||
with open(file, "w", encoding="utf-8") as f:
|
||||
f.write(page_source)
|
||||
|
||||
# Why is it so hard to just get the url of a single tab...
|
||||
# def get_fast_dl(driver,button):
|
||||
# windows = len(driver.window_handles)
|
||||
# return
|
||||
|
||||
# Because selenium seems to fuck up the url switching to a "download" tab,
|
||||
# I have to use the inbuilt download in chrome :(. That also means no etag/metadata
|
||||
# but to be honest it's using annotate-au.foundations.blackboard.com and not bbcswebdav system
|
||||
# NOTE: Switching to a "download" tab causes issues so we must use the in built
|
||||
# download in Chrome, which does not have etag or metadata information.
|
||||
# Files are using annotate-au.foundations.blackboard.com and not bbcswebdav system
|
||||
# so the tag may not exist in the first place.
|
||||
|
||||
|
||||
def download_file(dest):
|
||||
d = Path(DL_DIR)
|
||||
time.sleep(2)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,15 @@ from selenium.webdriver.support.wait import WebDriverWait
|
|||
from selenium.webdriver.support import expected_conditions as EC
|
||||
timeout = 5
|
||||
# find_element_safe = lambda name,timeout=30:WebDriverWait(driver, timeout).until(lambda x: x.find_element(By.ID, name))
|
||||
WaitClickable = lambda driver,locator:WebDriverWait(driver, timeout).until(EC.element_to_be_clickable(locator))
|
||||
WaitDiv = lambda driver,locator:WebDriverWait(driver, timeout).until(EC.presence_of_element_located(locator))
|
||||
SwitchToIFrame = lambda driver,locator:WebDriverWait(driver, timeout).until(EC.frame_to_be_available_and_switch_to_it(locator))
|
||||
|
||||
|
||||
def WaitClickable(driver, locator): return WebDriverWait(
|
||||
driver, timeout).until(EC.element_to_be_clickable(locator))
|
||||
|
||||
|
||||
def WaitDiv(driver, locator): return WebDriverWait(
|
||||
driver, timeout).until(EC.presence_of_element_located(locator))
|
||||
|
||||
|
||||
def SwitchToIFrame(driver, locator): return WebDriverWait(
|
||||
driver, timeout).until(EC.frame_to_be_available_and_switch_to_it(locator))
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user