Blackboard-marks/main.py

from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
# For chrome stuff
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.options import Options
# ---
from urllib.parse import parse_qs, urlparse
import os
import requests
import time
import getpass
import json
import re
import sys
import argparse
import pathlib

import utils.selectors
from utils.asset import Asset, RequestStack
from utils.wait import SwitchToIFrame, WaitClickable, WaitDiv
from constants.constants import BASE_URL
from utils.login import login
from utils.selectors import Selectors
from utils.utils import friendly_filename, get_assignment_name, get_text_excluding_children, save_html
import code
from random import randint

testing = False
try:
    testing = True
    from utils.test import get_etc  
except:
    def get_etc(*args): return False

cookie = None

def click_the_fing_button(driver,button):
    # https://stackoverflow.com/a/67414801 stupid bug
    try:
        ActionChains(driver).move_to_element(button)
        ActionChains(driver).click(button).perform()
        WebDriverWait(driver,2).until(EC.number_of_windows_to_be(2))
    except:
        driver.set_window_size(1024, 768)   # hack to wake selenium up when it doesnt want to click the button!
        click_the_fing_button(driver,button)
        driver.maximize_window()

# You can probably replace this with a recursive method like in blackboard scraper but tbh i just want to get this script done so i can stop working for once.
def scrape_further(driver,path):
    # attempts for bb-held tests
    attempts = driver.find_elements_by_xpath("//a[starts-with(@href, '/webapps/assessment')]")
    attempts = [ x.get_attribute('href') for x in attempts ]
    for i, attempt in enumerate(attempts):
        name = "attempt_"+str(i)+"_["+parse_qs(urlparse(attempt).query)['attempt_id'][0]+"]"
        attempt = re.sub("^"+BASE_URL,"",attempt)
        driver.execute_script("window.open('"+BASE_URL+attempt+"')")
        WebDriverWait(driver,10).until(EC.number_of_windows_to_be(3))
        driver.switch_to.window(driver.window_handles[2])
        save_html(path, name, driver.page_source)
        if testing:
            get_etc(driver, cookie, path)
        driver.close()
        driver.switch_to.window(driver.window_handles[1])

    # submission file for assignment
    request_stack = RequestStack(cookie)
    attempts = driver.find_elements_by_xpath("//a[starts-with(@href, '/webapps/assignment/download')]")
    attempts = [ x.get_attribute('href') for x in attempts ]
    for i, attempt in enumerate(attempts):
        request_stack.add_file(attempt,path)
    request_stack.download_all()

parser = argparse.ArgumentParser(description='Automated microsoft SSO login.')
# parser.add_argument("-p", "--password", help="Automatically use provided password", default="")
parser.add_argument("-u", "--username", help="Automatically use provided userID", default="")

args = parser.parse_args()

CAPABILITIES = DesiredCapabilities.CHROME
CAPABILITIES['goog:loggingPrefs'] = {'performance': 'ALL'}
OPTIONS = Options()
# OPTIONS.add_argument("--headless")
driver = webdriver.Chrome(
                            executable_path='chromedriver',
                            desired_capabilities=CAPABILITIES,
                            options=OPTIONS
                        )
driver.maximize_window()

cookie = {'Cookie': login(args, driver)} # do Login.

driver.get(BASE_URL+"/webapps/gradebook/do/student/viewCourses")

try:
    WaitClickable(driver,(By.CLASS_NAME, "button-1")).click()
except:
    print("no tos warning - skipped")
SwitchToIFrame(driver, (By.ID, 'mybbCanvas'))

# get courseIDs
courses = driver.find_element_by_id("left_stream_mygrades")\
                .find_elements_by_xpath("//div[@role='tab']")

course_details = []
for i, course_results in enumerate(courses):
    course_results = courses[i]
    ActionChains(driver).move_to_element(course_results).perform()
    course_url = course_results.get_attribute("bb:rhs")
    course_name = course_results.find_elements_by_xpath("//span[@class='stream_area_name']")[i].text
    course_name += " ["+parse_qs(urlparse(course_url).query)['course_id'][0]+"]"
    course_details.append({
        'name': course_name,
        'url' : course_url
    })

path = ['grades']
for i, course in enumerate(course_details):
    path.append(course['name']) # course name
    print(course['name'])
    driver.get(BASE_URL+course['url'])

    driver.execute_script("""
    mygrades.loadContentFrame = function(url) {
        window.open(url);
    }
    """)

    WaitClickable(driver,(By.XPATH,"//a[@value='A']")).click()

    table = driver.find_elements_by_xpath("//div[@id='grades_wrapper']/div")

    save_html("/".join(path), path[0], driver.page_source)

    for i, assignment in enumerate(table):
        print(i)
        buttons = assignment.find_elements_by_tag_name("input")
        block = None
        assignment_name = None
        information_link = False
        try:
            block = assignment.find_element_by_xpath("./div[@class='cell gradable']/a[@onclick]")
            information_link = True
        except:
            block = assignment.find_element_by_xpath("./div[@class='cell gradable']")
        assignment_name = get_assignment_name(driver,block)
        path.append(assignment_name)
        # download information if it exists.
        if information_link:
            ActionChains(driver).move_to_element(block).click(block).perform()
            print("Switched "+assignment_name)
            WebDriverWait(driver,10).until(EC.number_of_windows_to_be(2))
            driver.switch_to.window(driver.window_handles[1])
            save_html("/".join(path),"information",driver.page_source)
            scrape_further(driver, "/".join(path))
            driver.close()
            driver.switch_to.window(driver.window_handles[0])
        # download rubric if it exists.
        for button in buttons:
            action = button.get_attribute("onclick")
            if action != None and "showInLightBox" not in action:
                click_the_fing_button(driver,button)
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
                driver.switch_to.window(driver.window_handles[1])
                WaitDiv(driver, (By.CLASS_NAME, "rubricControlContainer"))
                save_html("/".join(path),"rubric",driver.page_source)
                driver.find_element_by_xpath("//li[@id='listViewTab']/a").click()
                WaitDiv(driver, (By.CLASS_NAME, "rubricGradingList"))
                save_html("/".join(path),"list",driver.page_source)
                detailed_buttons = driver.find_elements_by_xpath("//div[@class='u_controlsWrapper']/input")
                detailed_buttons[1].click()
                detailed_buttons[0].click()
                save_html("/".join(path),"list_detailed",driver.page_source)
                driver.close()
                driver.switch_to.window(driver.window_handles[0])
        path.pop()
    WaitClickable(driver,(By.XPATH,"//a[@value='S']")).click()
    save_html("/".join(path),"submitted",driver.page_source)
    try:
        WaitClickable(driver,(By.XPATH,"//div[@id='submissionReceipts']//a")).click()
        WaitClickable(driver,(By.XPATH,"//div[@id='listContainer_itemcount']//a[@class='pagelink']")).click()
    except:
        print('No items?')
    save_html("/".join(path),"receipts",driver.page_source)
    path.pop()


driver.quit()