|
|
本帖最后由 Lucker 于 2013-6-19 16:55 编辑
昨天在做一个网站的资源采集, 这里给大家分享出来
一定要提意见, 看看怎么改.....
首先说明: 俺是半罐罐水, 基础知识都知晓的一部分. 代码不够好大家请指教.
这里先发一段昨天晚上在XML解析过程中出现的一个问题. 在此感谢 ♂[渝]版主-Eagle 深夜对我的指导~
打包下载(完整压缩的7z,上传不了加了个后缀zip):
gather.7z.zip
(22.64 KB, 下载次数: 458)
资源地址屏蔽了的: 回复可见
程序代码(代码有删减,...字数超了的哇~~~~):
[code=csharp]
namespace gather
{
public partial class MainFrm : Form
{
public MainFrm()
{
InitializeComponent();
//数据添加到datagridview委托
_upDateStateDelegate = new UpDateDelegate(UpDateDgv);
//状态委托
_doUpStatus = new _upStatus(__doUpStatus);
}
//采集网址
string g_s = "回复可见1";
private object obj = new object();//用来锁定_np的.应该起作用了
private int _p = 0; //当前页 这个好像没有用
private int _ps = 0;//pagesize
private int _pc = 0;//pagecount总页数
private int _c = 0;//总记录数,recordcount
private int _nc = 0;//当前采集到的记录数nowcount,
private int _np = 0;//当前页数, nowpage
private int step = 0; //开始按钮用的, 用来执行 开始-> 暂停 <--> 继续
private int time = 0; //每个采集线程间隔时间
private int saveIndex = 0;//当前保存的数量
private int saveListCount = 20;//每次保存的数量
OleDbConnection con; //数据库连接
OleDbCommand cmd;//数据库操作
List<Thread> _threadMainList = new List<Thread>(); //采集线程
//空委托
private delegate void _voidDelegate();
//主线程委托
private delegate int getPageInfo();
//更新grid委托
private delegate void UpDateDelegate(d info);
private UpDateDelegate _upDateStateDelegate;
//设置状态委托 - 信息, 设置当前页, 设置当前总记录
private delegate void _upStatus(string status, bool np, bool nc);
private _upStatus _doUpStatus;
#region ----------------------------窗体事件 START----------------------------
//窗体开始
private void MainFrm_Load(object sender, EventArgs e)
{
string ConStr = "Provider=Microsoft.Jet.OLEDB.4.0;data source=" + Application.StartupPath + "\\data.mdb";//创建OleDbConnection对象
con = new OleDbConnection(ConStr);
try
{
con.Open();
if (con.State != ConnectionState.Open)
{
this.btn_start.Text = "DBERR";
this.btn_start.Enabled = false;
this.lbl_db.Text = "开启错误";
}
this.lbl_db.Text = "已连接";
}
catch { this.btn_start.Text = "DBERR"; this.btn_start.Enabled = false; this.lbl_db.Text = "开启错误"; }
finally { }
}
//按钮开始
private void btn_start_Click(object sender, EventArgs e)
{
//第一次开始 - 暂停
if (step == 0)
{
this.btn_start.Enabled = false;
getPageInfo _g = new getPageInfo(_threadGetPageInfo);
IAsyncResult ir = _g.BeginInvoke(null, null);
time = int.Parse(txt_zxjg.Text.ToString());
int init = _g.EndInvoke(ir);
if (init != 0)
{
this.btn_start.Enabled = true;
int count = Convert.ToInt32(txt_threadcount.Text.Trim());
if (count > _pc)
{
count = _pc;
}
_np = 1;//从第一页开始
for (int i = 0; i < count; i++)
{
Thread _temp = new Thread(new ThreadStart(delegate
{
_threadGetOnePage();
}));
_temp.Start();
_threadMainList.Add(_temp);
Thread.Sleep(Convert.ToInt32(txt_kqjg.Text.Trim()) * 1000);
}
timer_db.Enabled = true;
step = 1;
}
this.btn_start.Text = "暂停";
this.btn_start.Enabled = true;
}
else if (step == 1)
{
//停止 -> 继续
this.btn_start.Enabled = false;
foreach (Thread _t in _threadMainList)
{
try
{
_t.Suspend();
}
catch { }
}
this.lbl_status.Text = "已暂停...";
this.btn_start.Text = "继续";
this.btn_start.Enabled = false;
timer_db.Enabled = true;
step = 2;
}
else if (step == 2)
{
//继续 - > 暂停
this.btn_start.Enabled = false;
foreach (Thread _t in _threadMainList)
{
try
{
_t.Resume();
}
catch { }
}
this.btn_start.Text = "暂停";
timer_db.Enabled = true;
this.btn_start.Enabled = true;
step = 1;
}
}
//窗体关闭
private void MainFrm_FormClosing(object sender, FormClosingEventArgs e)
{
closeAll();
}
#endregion----------------------------窗体事件 END-------------------------------
#region ----------------------------时钟 START---------------------------------
//时钟
private void timer_db_Tick(object sender, EventArgs e)
{
int tmp_DataCount = this.dataGrid.Rows.Count - 1;
int tmp_saveListCount = saveListCount;
//判断条数是否够, 不够就动态改变, 保证每次Tick都保存
if ((saveIndex + tmp_saveListCount) > tmp_DataCount)
{
tmp_saveListCount = tmp_DataCount - saveIndex;
}
//有保存条数才保存没有就不保存
if (tmp_saveListCount > 0)
{
string _insertPreSql = "INSERT INTO [list{0}]([name],[type],[pic],[area],[des],[actor],[director],[url]) VALUES";
string _insertSql = "";
for (int i = 0; i < tmp_saveListCount; i++)
{
_insertSql = String.Format(_insertPreSql, (i % 5).ToString()) + _timerCreateSql(saveIndex++);
_timerInsert(_insertSql, saveIndex);//数据库插入
}
}
}
//创建SQL语句
private string _timerCreateSql(int index)
{
string tmpStr = "(";
tmpStr += "'" + this.dataGrid.Rows[index].Cells["名称"].Value.ToString() + "',";//name
tmpStr += "'" + this.dataGrid.Rows[index].Cells["类型"].Value.ToString() + "',";//type
tmpStr += "'" + this.dataGrid.Rows[index].Cells["图片"].Value.ToString() + "',";//pic
tmpStr += "'" + this.dataGrid.Rows[index].Cells["地区"].Value.ToString() + "',";//area
tmpStr += "'" + this.dataGrid.Rows[index].Cells["描述"].Value.ToString() + "',";//des
tmpStr += "'" + this.dataGrid.Rows[index].Cells["导演"].Value.ToString() + "',";//actor
tmpStr += "'" + this.dataGrid.Rows[index].Cells["主演"].Value.ToString() + "',";//director
tmpStr += "'" + this.dataGrid.Rows[index].Cells["资源"].Value.ToString() + "'";//url
tmpStr += ");";
return tmpStr;
}
//timer中执行添加到数据库
private void _timerInsert(string sql, int c)
{
try
{
cmd = new OleDbCommand(sql, con);
cmd.ExecuteNonQuery();
this.lbl_db.Text = "已插入" + c.ToString() + "项";
}
catch
{
this.timer_db.Enabled = false;
closeAll();
}
finally { cmd = null; }
}
#endregion------------------------------时钟 END---------------------------------
#region ----------------------------其他 START---------------------------------
//停止所有
private void closeAll()
{
timer_db.Enabled = false;
try
{
this.lbl_db.Text = "已关闭";
con.Close();
}
catch { }
foreach (Thread _t in _threadMainList)
{
try
{
if (_t.ThreadState == ThreadState.Suspended)
{
_t.Resume();
_t.Abort();
}
}
catch { }
}
}
#endregion--------------------------其他 END---------------------------------
}
}[/code]
|
|
窥视卡
雷达卡
发表于 2013-6-19 09:48:41
QQ好友和群
收藏
淘帖
提升卡
置顶卡
沉默卡
喧嚣卡
变色卡
千斤顶
显身卡
楼主
发表于 2013-6-19 10:30:02
