2
回答
为什么爬大众点评网会报403?有没有好心人告诉我
java.io.IOException: Server returned HTTP response code: 403 for URL: http://s.dianping.com/topic/6904156
	at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1839)
	at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1440)
	at com.fz.util.HttpClientUtils.connctionURL_Params(HttpClientUtils.java:104)
	at com.fz.util.HttpClientUtils.main(HttpClientUtils.java:215)


public class HttpClientUtils {
	private static Log log = LogFactory.getLog(HttpClientUtils.class);

	static Gson gson=new Gson();
    public HttpClientUtils() {
        // TODO Auto-generated constructor stub
    }
    
	public String connctionURLJson(String url, String method, String params) {
		StringBuffer bufferRes = new StringBuffer();
		url = url + params;
		try {
			URL realUrl = new URL(url);
			HttpURLConnection conn = (HttpURLConnection) realUrl.openConnection();
			// 请求方式
			conn.setRequestMethod(method);
			conn.setDoOutput(true);
			conn.setDoInput(true);
			conn.setUseCaches(false);
            conn.setInstanceFollowRedirects(true);
            conn.setRequestProperty("Content-Type", "application/json");
			conn.connect();

			InputStream in = conn.getInputStream();
			BufferedReader read = new BufferedReader(new InputStreamReader(in,"UTF-8"));
			String valueString = null;
			while ((valueString = read.readLine()) != null) {
				bufferRes.append(valueString);
			}
			read.close();
			in.close();
			in = null;
			if (conn != null) {
				// 关闭连接
				conn.disconnect();
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return bufferRes.toString();
	}
    

	public String connctionURL_Params(String url, String method, String params) {
		if (null == params ||StringUtils.isBlank(params)) {
			params="utf-8";
		}
		
		if (null == method ||StringUtils.isBlank(method)) {
			method="GET";
		}
		
		StringBuffer bufferRes = new StringBuffer();
		try {
			URL realUrl = new URL(url);
			HttpURLConnection conn = (HttpURLConnection) realUrl.openConnection();
			// 请求方式
			conn.setRequestMethod(method);
			conn.setDoOutput(true);
			conn.setDoInput(true);
			conn.setUseCaches(false);
            conn.setInstanceFollowRedirects(true);
            conn.setRequestProperty("Content-Type", "application/json");
//            conn.setRequestProperty("Charsert", "UTF-8");
//            conn.setRequestProperty("Charsert", "GBK");
//            conn.setRequestProperty("Charsert", "utf-8");
//            conn.setRequestProperty("connection", "keep-alive");
            
            conn.setRequestProperty(HttpMethodParams.USER_AGENT,"Mozilla/5.0 (Linux; U; Android 4.0; en-us; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30");
			conn.connect();

			InputStream in = conn.getInputStream();
			BufferedReader read = new BufferedReader(new InputStreamReader(in,params));
			String valueString = null;
			while ((valueString = read.readLine()) != null) {
				bufferRes.append(valueString);
			}
			read.close();
			in.close();
			in = null;
			if (conn != null) {
				// 关闭连接
				conn.disconnect();
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return bufferRes.toString();
	}
     
    /**
     * 
     * @param path
     * @param map
     * @param encode
     * @return
     */
     
    @SuppressWarnings("deprecation")
	public static String sendHttpClientPost(String path,
            Map<String, String> map, String encode) {
    	String code="";
        List<NameValuePair> list=new ArrayList<NameValuePair>();
        for(Map.Entry<String, String> entry:map.entrySet()){
             
            list.add(new BasicNameValuePair(entry.getKey(),entry.getValue()));
             
        }
        try {
            //实现将请求中的参数封装到请求参数中,请求体中
            UrlEncodedFormEntity  entity=new UrlEncodedFormEntity(list,encode);
            //使用post方式提交
            HttpPost httpPost=new HttpPost(path);
            httpPost.setEntity(entity);
            //指定post方式提交数据
            DefaultHttpClient client =new DefaultHttpClient();
             
            HttpResponse httpResponse=client.execute(httpPost);
            String g=gson.toJson(httpPost);
            log.info("--g--" +g);
            code=httpResponse.getStatusLine().getStatusCode()+"";
            log.info("--" + code);
            if(httpResponse.getStatusLine().getStatusCode()==200){
                return changeInputStream(httpResponse.getEntity().getContent(),encode);
            }
             
        } catch (Exception e) {
            // TODO: handle exception
        	log.info("-出现异常了-" +e);
             	e.printStackTrace();
        }
         
         
        return "code:" +code;
    }
    /**
     * 
     * @param inputStream
     * @param encode
     * @return
     */
    private static String changeInputStream(InputStream inputStream,
            String encode) {
        // TODO Auto-generated method stub
 
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        int len = 0;
        byte[] date = new byte[1024];
        String result = "";
        try {
 
            while ((len = inputStream.read(date)) != -1) {
 
                outputStream.write(date, 0, len);
            }
            result = new String(outputStream.toByteArray(), encode);
             
            return result;
 
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
 
        return null;
    }
    public static void main(String[] args) throws Exception {

//        String url="https://www.baidu.com/s?ie=UTF-8&wd=ip";
         
    	String url="http://s.dianping.com/topic/6904156";

       
    	try { 
    	   HttpClientUtils  htt=new HttpClientUtils();
    	   String res= htt.connctionURL_Params(url, "POST", "");
    	   log.info(res);
        }catch (Exception e) {
        	e.printStackTrace();
        }
    }
}





举报
陈all
发帖于2年前 2回/1K+阅
顶部