外贸电子商务网站建设平台,怎样开网店详细步骤,a站为什么不火了,xyz后缀的网站需求
找到最常用的200个协议 通过fofa搜索端口#xff0c;得到协议排名前五名和对应机器的数目。 遍历端口#xff0c;统计各个协议对应的机器数目#xff08;不准#xff0c;但能看出个大概#xff09;
读写API
API需要会员#xff0c;一天只能访问1000次。
import…需求
找到最常用的200个协议 通过fofa搜索端口得到协议排名前五名和对应机器的数目。 遍历端口统计各个协议对应的机器数目不准但能看出个大概
读写API
API需要会员一天只能访问1000次。
import base64
import urllib
from time import sleep
import requests
res {}
def onePort(j):text port str(j) text base64.b64encode(text.encode(utf-8)).decode(utf-8)text urllib.parse.quote(text)URL fhttps://fofa.info/api/v1/search/stats?fieldsprotocolqbase64{text}email*****key*****r requests.get(URL)response_dict r.json()print(当前端口为,j)print(response_dict)protocolsresponse_dict[aggs][protocol]for i in protocols:if i[name] in res:res[i[name]] res[i[name]] i[count]else:res[i[name]] i[count]print(res)for i in range(1,65535):onePort(i)sleep(10)爬虫
页面动态加载由于动态渲染的问题有的请求返回结果为空。
单线程未登录爬虫代码 import base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutorfrom selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
#直接添加这四行代码
from selenium.webdriver.chrome.options import Options
options Options()
options.add_argument(--headless)
# options.add_argument(--disable-gpu)failed[]
success[]
res {}
def onePort(j):s Service(r.\chromedriver.exe)driver webdriver.Chrome(services,optionsoptions)text port str(j) text base64.b64encode(text.encode(utf-8)).decode(utf-8)text urllib.parse.quote(text)print(text)driver.get(https://fofa.info/result?qbase64 text)sleep(7)page_text driver.page_source# print(page_text)tree etree.HTML(page_text)protos tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//a/text())nums tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//span/text())for i in range(len(protos)):protos[i] protos[i].strip( )protos[i] protos[i].strip(\n)protos[i] protos[i].strip( )nums[i] nums[i].strip( )nums[i] nums[i].strip(\n)nums[i] nums[i].strip( )nums[i] nums[i].replace(,, )nums[i] int(nums[i])if protos[i] in res:res[protos[i]] res[protos[i]] nums[i]else:res[protos[i]] nums[i]print(protos)print(nums)if len(protos) 0:failed.append(j)else:success.append(j)print(当前端口号, j)print(失败列表, failed)print(成功列表, success)print(res)driver.quit()for j in range(5000,10000):onePort(j)多线程未登录代码
一定要注意多线程同时读写问题全局变量上锁
import base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
import threading# 直接添加这四行代码
from selenium.webdriver.chrome.options import Optionsoptions Options()
options.add_argument(--headless)
# options.add_argument(--disable-gpu)failed []
success []
res {}
lock threading.Lock() # 创建线程锁def onePort(j):s Service(r.\chromedriver.exe)driver webdriver.Chrome(services, optionsoptions)text port str(j) text base64.b64encode(text.encode(utf-8)).decode(utf-8)text urllib.parse.quote(text)print(text)driver.get(https://fofa.info/result?qbase64 text)sleep(7)page_text driver.page_source# print(page_text)tree etree.HTML(page_text)protos tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//a/text())nums tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//span/text())with lock: # 使用线程锁保护对res变量的读写操作for i in range(len(protos)):protos[i] protos[i].strip( )protos[i] protos[i].strip(\n)protos[i] protos[i].strip( )nums[i] nums[i].strip( )nums[i] nums[i].strip(\n)nums[i] nums[i].strip( )nums[i] nums[i].replace(,, )nums[i] int(nums[i])if protos[i] in res:res[protos[i]] res[protos[i]] nums[i]else:res[protos[i]] nums[i]print(protos)print(nums)if len(protos) 0:failed.append(j)else:success.append(j)print(当前端口号, j)print(失败列表, failed)print(成功列表, success)print(res)driver.quit()with ThreadPoolExecutor(30) as t:for j in range(10000,10500):# 把下载任务提交给线程池t.submit(onePort, j)
手动登录获取cookie代码
# 填写webdriver的保存目录
s Service(r.\chromedriver.exe)
driver webdriver.Chrome(services)
# 记得写完整的url 包括http和https
driver.get(https://fofa.info)
# 程序打开网页后20秒内 “手动登陆账户”
time.sleep(20)
with open(cookies.txt,w) as f:# 将cookies保存为json格式f.write(json.dumps(driver.get_cookies()))driver.close()登录账号的单线程爬虫 from selenium import webdriver
import time
import jsonfrom selenium.webdriver.chrome.service import Serviceimport base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
from selenium.webdriver.chrome.options import Optionsfrom selenium.webdriver.chrome.options import Options
options Options()
# options.add_argument(--headless)
# options.add_argument(--disable-gpu)
options.add_argument(user-agentMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46)failed[]
success[]
res {}s Service(r.\chromedriver.exe)
driver webdriver.Chrome(services, optionsoptions)driver.get(https://fofa.info)
# 首先清除由于浏览器打开已有的cookies
driver.delete_all_cookies()with open(cookies.txt, r) as f:# 使用json读取cookies 注意读取的是文件 所以用load而不是loadscookies_list json.load(f)# 将expiry类型变为intfor cookie in cookies_list:# 并不是所有cookie都含有expiry 所以要用dict的get方法来获取if isinstance(cookie.get(expiry), float):cookie[expiry] int(cookie[expiry])driver.add_cookie(cookie)# 重新发送请求(这步是非常必要的要不然携带完cookie之后仍然在登录界面)
driver.get(https://fofa.info)
# sleep等待页面完全加载出来这一步很关键
time.sleep(3)j2
text port str(j)
text base64.b64encode(text.encode(utf-8)).decode(utf-8)
text urllib.parse.quote(text)
print(text)
sleep(10)
driver.get(https://fofa.info/result?qbase64 text)
sleep(6)
page_text driver.page_sourceprint(page_text)tree etree.HTML(page_text)protos tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//a/text())
nums tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//span/text())for i in range(len(protos)):protos[i] protos[i].strip( )protos[i] protos[i].strip(\n)protos[i] protos[i].strip( )nums[i] nums[i].strip( )nums[i] nums[i].strip(\n)nums[i] nums[i].strip( )nums[i] nums[i].replace(,, )nums[i] int(nums[i])if protos[i] in res:res[protos[i]] res[protos[i]] nums[i]else:res[protos[i]] nums[i]
print(protos)
print(nums)
if len(protos) 0:failed.append(j)
else:success.append(j)
print(当前端口号, j)
print(失败列表, failed)
print(成功列表, success)
print(res)driver.quit()登录用户多线程
import threadingfrom selenium import webdriver
import time
import jsonfrom selenium.webdriver.chrome.service import Serviceimport base64
import json
import urllib
from concurrent.futures import ThreadPoolExecutor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from lxml import etree
from time import sleep
from selenium.webdriver.chrome.options import Optionsfrom selenium.webdriver.chrome.options import Options
options Options()
options.add_argument(--headless)
# options.add_argument(--disable-gpu)
options.add_argument(user-agentMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.46)failed[]
success[]
res {}
lock threading.Lock()def onePort(j):s Service(r.\chromedriver.exe)driver webdriver.Chrome(services, optionsoptions)driver.get(https://fofa.info)# 首先清除由于浏览器打开已有的cookiesdriver.delete_all_cookies()with open(cookies.txt, r) as f:# 使用json读取cookies 注意读取的是文件 所以用load而不是loadscookies_list json.load(f)# 将expiry类型变为intfor cookie in cookies_list:# 并不是所有cookie都含有expiry 所以要用dict的get方法来获取if isinstance(cookie.get(expiry), float):cookie[expiry] int(cookie[expiry])driver.add_cookie(cookie)# 重新发送请求(这步是非常必要的要不然携带完cookie之后仍然在登录界面)driver.get(https://fofa.info)# sleep等待页面完全加载出来这一步很关键time.sleep(3)text port str(j) text base64.b64encode(text.encode(utf-8)).decode(utf-8)text urllib.parse.quote(text)print(text)driver.get(https://fofa.info/result?qbase64 text)sleep(6)page_text driver.page_sourcetree etree.HTML(page_text)protos tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//a/text())nums tree.xpath(//div[classhsxa-ui-component hsxa-meta-data-statistical-list hsxa-pos-rel]/div[13]//li//span/text())with lock: # 使用线程锁保护对res变量的读写操作for i in range(len(protos)):protos[i] protos[i].strip( )protos[i] protos[i].strip(\n)protos[i] protos[i].strip( )nums[i] nums[i].strip( )nums[i] nums[i].strip(\n)nums[i] nums[i].strip( )nums[i] nums[i].replace(,, )nums[i] int(nums[i])if protos[i] in res:res[protos[i]] res[protos[i]] nums[i]else:res[protos[i]] nums[i]print(protos)print(nums)if len(protos) 0:failed.append(j)else:success.append(j)print(当前端口号, j)print(失败列表, failed)print(成功列表, success)print(res)driver.quit()with ThreadPoolExecutor(1) as t:for j in range(3679,4000):# 把下载任务提交给线程池t.submit(onePort, j)
