Skip to content

Commit

Permalink
NI/JO started debugging Search
Browse files Browse the repository at this point in the history
  • Loading branch information
@nathan-inge
nathan-inge committed Feb 16, 2018
1 parent ec5a25b commit 05b3f6d
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 56 deletions.
105 changes: 52 additions & 53 deletions src/edu/ucsb/cs56/projects/scrapers/ucsb_curriculum/UCSBCurriculumSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public int loadCourses(String dept, String qtr, String level) throws Exception {


String page = getPage(dept,qtr,level);
String origpage = page;
String origpage = page;
int num_lectures = 0;

String search_string = "<tr class=\"CourseInfoRow\">";
Expand All @@ -185,22 +185,21 @@ public int loadCourses(String dept, String qtr, String level) throws Exception {

ArrayList<String> lecture_html = new ArrayList<String>();

try {

page = page.substring(0, page.lastIndexOf("</table>"));
//next line causes problems
}
catch (Exception e) {
throw new Exception ("webpage did not have expected structure"+origpage);
}
try {
page = page.substring(0, page.lastIndexOf("</table>"));
//next line causes problems
}
catch (Exception e) {
throw new Exception ("webpage did not have expected structure"+origpage);
}

while(course_pos != -1){

String lect = "";
if (next_course_pos == -1){
lect += page.substring(course_pos);
}
else{
} else{
lect += page.substring(course_pos, next_course_pos);
}
lecture_html.add(lect);
Expand Down Expand Up @@ -233,54 +232,54 @@ public int loadCourses(String dept, String qtr, String level) throws Exception {
@return ArrayLIst of Strings of offered subject areas
*/
public ArrayList<String> findSubjectAreas(String html){
ArrayList<String> SubjectAreas = new ArrayList<String>();
String before_list_string = "<select name=\"ctl00$pageContent$courseList\" id=\"ctl00_pageContent_courseList\" class=\"droplist\">";
String after_list_string = "</select>";
String all_subjects = "";
String[] all_subjects_split;
try{
all_subjects = html.substring(html.indexOf(before_list_string)+ before_list_string.length(),html.indexOf(after_list_string) );
// System.out.println(all_subjects);

}catch (Exception e){
System.err.println("The HTML of UCSB Curriculum Serach has changed.");
System.err.println("This scraper must be updated.");
}
all_subjects_split = all_subjects.split("\n");
for(int i = 1; i < all_subjects_split.length; i++){
// System.out.println(all_subjects_split[i]);
String temp = all_subjects_split[i].substring(all_subjects_split[i].lastIndexOf("=")+2, all_subjects_split[i].lastIndexOf("\""));
SubjectAreas.add(temp);
// System.out.println(temp);
}

return SubjectAreas;
ArrayList<String> SubjectAreas = new ArrayList<String>();
String before_list_string = "<select name=\"ctl00$pageContent$courseList\" id=\"ctl00_pageContent_courseList\" class=\"droplist\">";
String after_list_string = "</select>";
String all_subjects = "";
String[] all_subjects_split;
try{
all_subjects = html.substring(html.indexOf(before_list_string)+ before_list_string.length(),html.indexOf(after_list_string) );
// System.out.println(all_subjects);
}catch (Exception e){
System.err.println("The HTML of UCSB Curriculum Serach has changed.");
System.err.println("This scraper must be updated.");
}
all_subjects_split = all_subjects.split("\n");
for(int i = 1; i < all_subjects_split.length; i++){
// System.out.println(all_subjects_split[i]);
String temp = all_subjects_split[i].substring(all_subjects_split[i].lastIndexOf("=")+2, all_subjects_split[i].lastIndexOf("\""));
SubjectAreas.add(temp);
// System.out.println(temp);
}
return SubjectAreas;
}
/** Find the list of quarters that are viewable given the HTML
@param html HTML of the main URL
@return ArrayList of Strings of quarters
*/
public ArrayList<String> findQuarterAndYear(String html){
ArrayList<String> availableQuarters = new ArrayList<String>();
String before_list_string = "<select name=\"ctl00$pageContent$quarterList\" id=\"ctl00_pageContent_quarterList\" class=\"droplist\">";
String after_list_string = "<option value=\"20154\">FALL 2015 </option>";
String all_quarters = "";
String[] all_quarters_split;
try{
all_quarters = html.substring(html.indexOf(before_list_string)+ before_list_string.length(),html.indexOf(after_list_string)+after_list_string.length());
}catch (Exception e){
System.err.println("The HTML of UCSB Curriculum Serach has changed.");
System.err.println("This scraper must be updated.");
}
all_quarters_split = all_quarters.split("\n");
for(int i = 1; i < all_quarters_split.length; i++)
{
int startQuarterName = all_quarters_split[i].indexOf(">") + 1;
String temp = all_quarters_split[i].substring(startQuarterName, all_quarters_split[i].lastIndexOf("<"));
temp = temp.trim();
availableQuarters.add(temp);
}
return availableQuarters;
ArrayList<String> availableQuarters = new ArrayList<String>();
String before_list_string = "<select name=\"ctl00$pageContent$quarterList\" id=\"ctl00_pageContent_quarterList\" class=\"droplist\">";
String after_list_string = "<option value=\"20154\">FALL 2015 </option>";
String all_quarters = "";
String[] all_quarters_split;
try{
all_quarters = html.substring(html.indexOf(before_list_string)+ before_list_string.length(),html.indexOf(after_list_string)+after_list_string.length());
}catch (Exception e){
System.err.println("The HTML of UCSB Curriculum Serach has changed.");
System.err.println("This scraper must be updated.");
}
all_quarters_split = all_quarters.split("\n");
for(int i = 1; i < all_quarters_split.length; i++)
{
int startQuarterName = all_quarters_split[i].indexOf(">") + 1;
String temp = all_quarters_split[i].substring(startQuarterName, all_quarters_split[i].lastIndexOf("<"));
temp = temp.trim();
availableQuarters.add(temp);
}
return availableQuarters;
}


Expand Down
6 changes: 3 additions & 3 deletions src/edu/ucsb/cs56/projects/scrapers/ucsb_curriculum/UCSBCurriculumSearchTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public void test_loadCourses() {
final String qtr = "20112"; // 2012 = S11 [yyyyQ, where Q is 1,2,3,4 (1=W, 2=S, 3=M, 4=F)]
final String level = "Undergraduate"; // other options: "Graduate", "All".

String page = uccs.getPage(dept,qtr,level);
//String page = uccs.getPage(dept,qtr,level);

int num_courses = uccs.loadCourses(dept, qtr, level);

Expand All @@ -43,10 +43,10 @@ public void test_loadCourses2() {

UCSBCurriculumSearch uccs2 = new UCSBCurriculumSearch();
final String dept = "JAPAN"; //the department
final String qtr = "20161"; //2016 Winter quarter
final String qtr = "20112"; //2016 Winter quarter
final String level = "Undergraduate";

String page = uccs2.getPage(dept,qtr,level);
//String page = uccs2.getPage(dept,qtr,level);

int num_courses = uccs2.loadCourses(dept, qtr, level);

Expand Down

0 comments on commit 05b3f6d

Please sign in to comment.