add selects and textareas in form extraction & submission tutorial

Author: x4nth055

web-scraping/extract-and-fill-forms/form_extractor.py

@@ -21,7 +21,9 @@ def get_form_details(form):
  including action, method and list of form controls (inputs, etc)"""
 details={}
 # get the form action (requested URL)
-action=form.attrs.get("action").lower()
+action=form.attrs.get("action")
+ifaction:
+action=action.lower()
 # get the form method (POST, GET, DELETE, etc)
 # if not specified, GET is the default in HTML
 method=form.attrs.get("method", "get").lower()
@@ -36,6 +38,38 @@ def get_form_details(form):
 input_value=input_tag.attrs.get("value", "")
 # add everything to that list
 inputs.append({"type": input_type, "name": input_name, "value": input_value})
+forselectinform.find_all("select"):
+# get the name attribute
+select_name=select.attrs.get("name")
+# set the type as select
+select_type="select"
+select_options= []
+# the default select value
+select_default_value=""
+# iterate over options and get the value of each
+forselect_optioninselect.find_all("option"):
+# get the option value used to submit the form
+option_value=select_option.attrs.get("value")
+ifoption_value:
+select_options.append(option_value)
+ifselect_option.attrs.get("selected"):
+# if 'selected' attribute is set, set this option as default 
+select_default_value=option_value
+ifnotselect_default_valueandselect_options:
+# if the default is not set, and there are options, take the first option as default
+select_default_value=select_options[0]
+# add the select to the inputs list
+inputs.append({"type": select_type, "name": select_name, "values": select_options, "value": select_default_value})
+fortextareainform.find_all("textarea"):
+# get the name attribute
+textarea_name=textarea.attrs.get("name")
+# set the type as textarea
+textarea_type="textarea"
+# get the textarea value
+textarea_value=textarea.attrs.get("value", "")
+# add the textarea to the inputs list
+inputs.append({"type": textarea_type, "name": textarea_name, "value": textarea_value})
+
 # put everything to the resulting dictionary
 details["action"] =action
 details["method"] =method

web-scraping/extract-and-fill-forms/form_submitter.py

@@ -1,5 +1,4 @@
 frombs4importBeautifulSoup
-fromrequests_htmlimportHTMLSession
 
 frompprintimportpprint
 fromurllib.parseimporturljoin
@@ -10,25 +9,50 @@
 
 # get the URL from the command line
 url=sys.argv[1]
+all_forms=get_all_forms(url)
 # get the first form (edit this as you wish)
-first_form=get_all_forms(url)[0]
+# first_form = get_all_forms(url)[0]
+fori, finenumerate(all_forms, start=1):
+form_details=get_form_details(f)
+print(f"{i} #")
+pprint(form_details)
+print("="*50)
+
+choice=int(input("Enter form indice: "))
 # extract all form details
-form_details=get_form_details(first_form)
+form_details=get_form_details(all_forms[choice-1])
 pprint(form_details)
 # the data body we want to submit
 data={}
 forinput_taginform_details["inputs"]:
 ifinput_tag["type"] =="hidden":
 # if it's hidden, use the default value
 data[input_tag["name"]] =input_tag["value"]
+elifinput_tag["type"] =="select":
+fori, optioninenumerate(input_tag["values"], start=1):
+# iterate over available select options
+ifoption==input_tag["value"]:
+print(f"{i} # {option} (default)")
+else:
+print(f"{i} # {option}")
+choice=input(f"Enter the option for the select field '{input_tag['name']}' (1-{i}): ")
+try:
+choice=int(choice)
+except:
+# choice invalid, take the default
+value=input_tag["value"]
+else:
+value=input_tag["values"][choice-1]
+data[input_tag["name"]] =value
 elifinput_tag["type"] !="submit":
 # all others except submit, prompt the user to set it
 value=input(f"Enter the value of the field '{input_tag['name']}' (type: {input_tag['type']}): ")
 data[input_tag["name"]] =value
+
 
 # join the url with the action (form request URL)
 url=urljoin(url, form_details["action"])
-
+# pprint(data)
 ifform_details["method"] =="post":
 res=session.post(url, data=data)
 elifform_details["method"] =="get":