Table of Contents
创建滑动解锁页面
准备三张图片
- 背景图片
- 缺口图片
- 滑块图片
使用NEXTJS构建页面
const SliderCaptcha = ({ onVerify }: SliderCaptchaProps) => {
const [isDragging, setIsDragging] = useState(false)
const [stPos,setStPos] = useState(0)
const [position, setPosition] = useState(0)
const containerRef = useRef(null)
const handleStart = (e) => {
console.log(e)
setIsDragging(true)
}
const handleMove = (e) => {
if (isDragging && containerRef.current) {
let clientX = e.clientX || (e.touches && e.touches[0].clientX)
console.log('move', clientX)
setPosition(clientX)
}
}
const handleEnd = () => {
setIsDragging(false)
const correctX = 120
if (Math.abs(position - correctX) < 10) {
onVerify(true)
} else {
onVerify(false)
}
}
return (
<div
className="flex items-center justify-center"
ref={containerRef}
onMouseMove={handleMove}
onMouseUp={handleEnd}
onMouseLeave={handleEnd}
onTouchMove={handleMove}
onTouchEnd={handleEnd}
>
<div className="relative flex" style={{ width: '265px', height: '170px' }}>
<img src="/pavement.jpeg" />
<img
src="/puzzle_piece.png"
style={{ position: 'relative', top: '-170px', left: `${position}px` }}
/>
</div>
<div onMouseDown={handleStart} style={{width:'265px'}}>
<img
id="move-img"
src="/fast_forward.png"
style={{ position: 'relative', top: '0px',left:`${position}px`, width: '45px', height: '45px' }}
/>
</div>
</div>
)
}
export { SliderCaptcha }创建识别服务
-
利用yolov8,label-studio 训练模型，识别背景缺口位置
-
搭建识别服务python + flask
!/usr/bin/env python3
"""Simple Flask server for captcha detect."""
Importing flask module in the project is mandatory
An object of Flask class is our WSGI application.
from flask import Flask
from flask import request, jsonify
import cv2
import numpy as np
import base64
import onnxruntime as ort
import json
Load the ONNX model
ONNX_MODEL_PATH = "runs/train/exp/weights/best.onnx"
session = ort.InferenceSession(ONNX_MODEL_PATH)
Get model input details
input_name = session.get_inputs()[0].name
input_shape = session.get_inputs()[0].shape # e.g., [1, 3, 320, 320]
img_size = input_shape[2] # Assuming square input (e.g., 320)
Class names (from your data.yaml)
class_names = ["target"] # Replace with your classes
Preprocess the image (unchanged)
def preprocess_image(img, img_size):
"""Preprocess image function."""
orig_shape = img.shape[:2] # (height, width)
ratio = min(img_size / orig_shape[0], img_size / orig_shape[1])
new_shape = (int(orig_shape[1] * ratio), int(orig_shape[0] * ratio))
img_resized = cv2.resize(img, new_shape, interpolation=cv2.INTER_LINEAR)
img_padded = np.zeros((img_size, img_size, 3), dtype=np.uint8)
top = (img_size - new_shape[1]) // 2
left = (img_size - new_shape[0]) // 2
img_padded[top : top + new_shape[1], left : left + new_shape[0]] = img_resized
img_input = img_padded.astype(np.float32) / 255.0
img_input = img_input.transpose(2, 0, 1)
img_input = np.expand_dims(img_input, axis=0)
return img_input, orig_shape, ratio, (top, left)
def postprocess_output(
output, orig_shape, ratio, padding, conf_thres=0.25, iou_thres=0.45
):
print("Raw output shape:", output.shape)
# Transpose to [n_detections, 5] predictions = output[0].T # [2100, 5] print("Predictions shape:", predictions.shape) # Extract boxes and confidence boxes = predictions[:, :4] # [x_center, y_center, width, height] conf = predictions[:, 4] # Confidence scores # Filter by confidence mask = conf > conf_thres boxes = boxes[mask] scores = conf[mask] # Use conf as scores since no class probs print("Filtered detections:", boxes.shape[0]) # Handle case with no detections if boxes.shape[0] == 0: print("No detections above confidence threshold.") return [] # For single-class or no class scores, assume class_id = 0 (or adjust if multi-class elsewhere) class_ids = np.zeros(len(scores), dtype=int) # Default to class 0 # NMS (Non-Max Suppression) indices = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), conf_thres, iou_thres) # Adjust boxes to original image coordinates detections = [] top, left = padding for i in indices: box = boxes[i] score = scores[i] class_id = class_ids[i] x_center, y_center, w, h = box x = (x_center - w / 2 - left) / ratio y = (y_center - h / 2 - top) / ratio w /= ratio h /= ratio x = max(0, min(x, orig_shape[1])) y = max(0, min(y, orig_shape[0])) w = min(w, orig_shape[1] - x) h = min(h, orig_shape[0] - y) detections.append((class_names[class_id], score, (x, y, w, h))) return detectionsMain inference function (unchanged)
def detect_image(img):
"""Detect image function."""
img_input, orig_shape, ratio, padding = preprocess_image(img, img_size)
outputs = session.run(None, {input_name: img_input})
detections = postprocess_output(outputs[0], orig_shape, ratio, padding)
result = {"rects": [], "img": ""}
img = cv2.imread(image_path)
for class_name, score, (x, y, w, h) in detections:
x, y, w, h = int(x), int(y), int(w), int(h)
cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
label = f"{class_name}: {score:.2f}"
cv2.putText(
img, label, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2
)
result["rects"].append((x, y, w, h))
_, buffer = cv2.imencode(".jpeg", img)
result["img"] = base64.b64encode(buffer).decode("utf-8")
cv2.imwrite("./detect1.jpeg", img)
print(f"Saved detection result to {'./detect.jpeg'}")
return jsonify(result)
app = Flask(name)
@app.route("/")
def hello_world():
"""Hello_World function."""
print("Hello World")
return "Hello World"
@app.route("/detect", methods=["POST"])
def detect_captcha():
"""Detect captcha function."""
print("detect_captcha")
base64_string = request.get_json()
base64_string.decode("utf-8").split("base64,")[1].strip("')")
img_bytes = base64.b64decode(base64_string["data"])
# Convert bytes to a NumPy array img_array = np.frombuffer(img_bytes, np.uint8) # Decode the image using cv2.imdecode image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) return detect_image(image)if name == "main ":
app.run(debug=True, port=5001)
使用puppeteer 工具在网页上滑动验证
async sloveCaptcha(page:Page,tryTimes:number){
if(tryTimes<=0){
console.log('sloved failed')
return
}
const captcha_modal = await page.$('div#captcha_modal');
const cmBox = await captcha_modal?.boundingBox();
console.log(`cmBox ${JSON.stringify(cmBox)}`)
const cpc_img = await page.$('#cpc_img');
const cpcBox = await cpc_img?.boundingBox()
if(!cpcBox){
console.error('Get cpcBox failed')
return;
}
console.log(`cpcBox ${JSON.stringify(cpcBox)}`)
const cpc_image = await cpc_img?.evaluate((el) => {
return el
.getAttribute("src")
?.replace("data:image/jpg;base64,", "");
});
if (!cpc_image) return;
const puzz = await axios
.post("http://localhost:5001/detect", { data: cpc_image })
.then((res) => {
const result = res.data as { rects: any; img: string };
console.log(result["rects"]);
fspromises.writeFile(
"./cache/detected.jpeg",
result.img,
"base64",
);
return result.rects[0] || [];
})
.catch((err) => {
console.error(err);
return [];
});
if (puzz.length === 0) return;
console.log(`puzzz ${JSON.stringify(puzz)}`);
//page.mouse.move(x, y)
const destCx = cpcBox.x+puzz[0]+puzz[2]/2;
const moveHandler = await page.$("img.move-img");
if (!moveHandler) {
console.log("move img not found!");
return;
}
const moveBox= await moveHandler.boundingBox();
if (!moveBox) {
console.log("bounding box error");
return;
}
let xPos = moveBox.x + moveBox.width / 2;
let yPos = moveBox.y + moveBox.height / 2;
console.log(`entered to move img ${xPos},${yPos}`);
await page.mouse.move(xPos, yPos,{steps:5});
await this.delay(300)
await page.mouse.down();
await this.delay(200);
//await page.mouse.move(destCx+1.5,yPos,{steps:5})
while(xPos <= destCx){
const randY = getRandomInt(-1, 1)
console.log(`move to ${xPos}, ${yPos+randY}`);
await page.mouse.move(xPos, yPos+randY)
await this.delay(getRandomInt(10, 50))
xPos += getRandomInt(10,30);
}
await this.delay(200)
await page.mouse.move(destCx, yPos,{steps:5})
await page.mouse.up();
await this.delay(1000)
if(page.url().includes('privatedomain')){
this.sloveCaptcha(page, tryTimes-1)
}else{
console.log(`solved captcha ${page.url()}`)
}
}
}NOTES:
-
识别服务返回的缺口位置是相对与图片的坐标，需要转换成网页的坐标
const destCx = cpcBox.x+puzz[0]+puzz[2]/2; -
滑动滑块的时候需要微调每次滑动的距离
xPos += getRandomInt(10,30);