curl-users
cURL & Monster.com
Date: Wed, 13 Jul 2005 16:58:03 -0700
Hello. This is my first post to the cURL community. I love the tool and am
relatively new to it. I just had a question that I hope'd some more
experienced users could help me out with. I'm trying to get cURL to run a
search for me on monster.com. I understand how to use cookies with the -c
and -b options but monster places some cookie information that i am some how
missing. I think it has to do with the javascript on thier site-
unfortunately im not the best at reading java. anyways, here is my cURL
command:
"curl -c /dir/dir2/monster.txt -L -D /dir/dir2/header.txt
http://www.monster.com"
this yields the following in the monster.txt cookie file:
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Netscape HTTP Cookie File
# http://www.netscape.com/newsref/std/cookie_spec.html
# This file was generated by libcurl! Edit at your own risk.
.monster.com TRUE / FALSE 1136850018 ez 95101-95103%2b95106%2b95108-95139%2b95141-95142%2b95148%2b95150-95161%2b95164%2b95170-95173%2b95190-95194%2b95196
www.monster.com FALSE / FALSE 1134438018 mnshp3 2
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
and the header.txt file (just for information purposes):
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
HTTP/1.0 200 OK
Server: Microsoft-IIS/6.0
Last-Modified: Wed, 13 Jul 2005 23:40:18 GMT
Content-Length: 46138
Content-Type: text/html
Set-Cookie:
ez=95101-95103%2b95106%2b95108-95139%2b95141-95142%2b95148%2b95150-95161%2b95164%2b95170-95173%2b95190-95194%2b95196;
expires=Mon, 09 Jan 2006 23:40:18 GMT; path=/; domain=.monster.com
Set-Cookie: mnshp3=2; expires=Tue, 13 Dec 2005 01:40:18 GMT; path=/;
domain=www.monster.com;
Expires: Wed, 13 Jul 2005 23:40:18 GMT
Cache-Control: max-age=0, no-cache
Pragma: no-cache
Date: Wed, 13 Jul 2005 23:40:18 GMT
Connection: keep-alive
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Now, when i visit the page in my browser (firefox) the cookie entries that
are created are as follows:
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# HTTP Cookie File
# http://www.netscape.com/newsref/std/cookie_spec.html
# This is a generated file! Do not edit.
# To delete cookies, use the Cookie Manager.
ads.monster.com FALSE / FALSE 2145844947 NGUserID a0a0a1f-3496-1121298195-5
.monster.com TRUE / FALSE 1436701542 COOKIE_ID 67.172.177.197-4192796112.29722628
.monster.com TRUE / FALSE 1136893541 ez 95101-95103%2b95106%2b95108-95139%2b95141-95142%2b95148%2b95150-95161%2b95164%2b95170-95173%2b95190-95194%2b95196
.atdmt.com TRUE / FALSE 1278936003 AA002 001121298340-1040215584/1122507940
.www.monster.com TRUE / FALSE 1134481541 mnshp3 2
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
I think that
".monster.com TRUE / FALSE 1436701542 COOKIE_ID 67.172.177.197-4192796112.29722628"
is the magic line im missing from my cookie file.
I've tried just copy/pasting =) didnt work.
There is javascript that I think dynamically creates what I am missing from
cURL as far as the cookie goes. SO. My real question is - how can i get cURL
to flow through the page and capture all cookie content even if it is not
all created within the header as I am suspecting it is not.
Here is the portion of the monster.com source code that i do not understand
how to make cURL accomodate - look at the java cuz i think thats where the
key to this mystery will be found.
And I'd like to also say THANK YOU to anyone that can help! I hope to help
others and participate in the cURL community, its one of the coolest tools
i've ever found.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<script type="text/javascript" language="javascript">
<!-- //
if (self != top) top.location.href = window.location.href;
// -->
</script>
<script language="JavaScript1.1">
<!--
function dcsTrk(dcs_src)
{
if (document.images)
{
dcsTrk.dcs_img[dcsTrk.dcs_ptr] = new Image;
dcsTrk.dcs_img[dcsTrk.dcs_ptr].src = dcs_src;
dcsTrk.dcs_ptr++;
}
}
dcsTrk.dcs_img = new Array;
dcsTrk.dcs_ptr = 0;
function dcsMeta_ex(dcsEXTv)
{
var F=false;
var myDocumentElements;
if (document.all)
{
F = true;
myDocumentElements=document.all.tags("meta");
}
if (!F && document.documentElement)
{
F = true;
myDocumentElements=document.getElementsByTagName("meta");
}
if (F)
{
for (var i=1; i<=myDocumentElements.length;i++)
{
myMeta=myDocumentElements.item(i-1);
if (myMeta.name.indexOf('WT.mns_')==0)
dcsEXTv+="&"+myMeta.name+"="+escape(myMeta.content);
}
}
return dcsEXTv
}
function dcs_2_0_ex(dcs_URI,dcs_QRY,dcs_EXT)
{
var dCurrent = new Date();
var P = "";
P+="http"+(window.location.protocol.indexOf('https:')==0?'s':'')+"://cookie.monster.com/DCS000007_2Q9M/dcs.gif?";
if (typeof(dcs_URI) == "string") P+="dcsuri="+escape(dcs_URI);
else P+= "dcsuri=/invalidjavascript";
if (typeof(dcs_QRY) == "string") P+="&dcsqry="+escape(dcs_QRY);
if ((window.document.referrer != "") && (window.document.referrer != "-"))
{
if (!(navigator.appName == "Microsoft Internet Explorer" &&
parseInt(navigator.appVersion) < 4) )
{
P+="&dcsref="+escape(window.document.referrer);
}
}
if (typeof(dcs_EXT) == "string") P+=dcs_EXT;
P+="&dcssip=www.monster.com";
P+="&dcsdat="+escape(dCurrent.getTime());
dcsTrk(P);
}
function outEdge(dcs_QRY)
{
outEdgeCore(dcs_QRY);
window.location=dcs_QRY;
}
function outEdgeCore(dcs_QRY)
{
var dcsURI=window.location.pathname;
var dcsQRY=window.location.search;
var dcsEXT="&TMP.url=" + escape(dcs_QRY);
dcsEXT = dcsMeta_ex(dcsEXT);
dcs_2_0_ex(dcsURI,dcsQRY,dcsEXT);
}
//-->
</script>
<!-- http_time($time() + 13140000) makes the cookie a perm cookie set to
expire in 1 yr -->
<link rel="stylesheet" type="text/css" href="/includes/styles05.css" />
<script src='/includes/MyScript.js'></script>
<style>
.ddimagetabs{
dispaly:table;
}
.tabcontentcontainer{
width:100%;
height:1px;
background-color: #660099;
text-align: justify;
}
.tabcontent{
position: absolute;
font-family: Verdana, Arial, Helvetica, sans-serif;
font-size: 10px;
font-weight: bold;
color: #000066;
text-decoration: none;
display:none;
z-index: 999999;
width:100%;
}
.mytable
{
position: absolute;
border-color: #669933; /*#d8dce5;*/
border-style: solid;
border-width: 1;
/*filter:alpha(opacity=80, finishopacity=300, style=1, startx=0, starty=0,
finishx=140, finishy=270);
-moz-opacity:.9;*/
background: #E3E7DB;
z-index:99999;
}
.ddimagetabs a span.left
{
width:107px;
PADDING-LEFT: 19px;
text-align: left;
padding-top: 2px;
BACKGROUND-IMAGE:
url(http://media.monster.com/mm/usen/home/05/tab_left.gif);
cursor: hand;
background-repeat: no-repeat;
TEXT-DECORATION: none;
}
.ddimagetabs a span.middle
{
width:118px;
PADDING-LEFT: 21px;
padding-top: 2px;
text-align: left;
BACKGROUND-IMAGE:
url(http://media.monster.com/mm/usen/home/05/tab_middle2.gif);
cursor: hand;
background-repeat: no-repeat;
TEXT-DECORATION: none;
}
.ddimagetabs a span.right
{
width:120px;
PADDING-LEFT: 21px;
padding-top: 2px;
text-align: left;
BACKGROUND-IMAGE:
url(http://media.monster.com/mm/usen/home/05/tab_end1.gif);
cursor: hand;
background-repeat: no-repeat;
TEXT-DECORATION: none;
}
.ddimagetabs span {
float: left;
display: inline;
PADDING-LEFT: 5px;
PADDING-BOTTOM: 1px;
}
.ddimagetabs a{
display: table-cell;
TEXT-DECORATION: none;
FONT-FAMILY: Verdana;
FONT-WEIGHT: bold;
FONT-SIZE: 12px;
COLOR: #663399;
height: 18px; /*height of tab image */
line-height: 15px;
float: left;
PADDING-LEFT: 0px;
/*padding-top: 2px; /*vertical offset of tab text from top of tab*/
/*BACKGROUND-IMAGE:
url(http://media.monster.com/mm/usen/home/05/navbutton-on.gif); */
background-repeat: no-repeat;
text-align: left;
vertical-align: middle;
PADDING-BOTTOM: 0px;
}
.frmClass
{
visibility:hidden;
position:absolute;
border-style:hidden;
border:none;
top:0px;
left:0px;
width:0px;
height:0px;
z-index:98;
filter:alpha(opacity=80, finishopacity=300, style=1, startx=0, starty=0,
finishx=140, finishy=270);
-moz-opacity:.9;
}
.WHTBig{
FONT-SIZE: small; COLOR: #ffffff; FONT-STYLE: normal; FONT-FAMILY: Verdana,
Geneva, Arial, san-serif;FONT-WEIGHT: bold;
}
.purpleHd {
FONT-SIZE: 18px; COLOR: #660099; LINE-HEIGHT: 24px; FONT-FAMILY: Verdana,
Geneva, Arial, san-serif; LETTER-SPACING: -1px
}
.BorderStyle2 {
BORDER-RIGHT: #0161b2 1px solid;BORDER-TOP: #0161b2 0px solid;BORDER-LEFT:
#0161b2 1px solid; BORDER-BOTTOM: #0161b2 1px solid
}
.Smallbold1 {
FONT-WEIGHT: bold; FONT-SIZE: 9px; FONT-STYLE: normal; FONT-FAMILY:
Verdana, Geneva, Arial, san-serif
}
.LoginLinks {
font-family:verdana,arial,helvetica,sans-serif;
font-weight:bold;
font-size:12px;
text-decoration:none;
color:black;
}
.LoginText {
font-family:verdana,arial,helvetica,sans-serif;
font-weight:bold;
font-size:11px;
text-decoration:none;
color:black;
}
</style>
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
that portion is in the <head> tag and this is near the bottom in the <body>
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<script>
activeMenuParent = 0
activeMenu = 0
curMenu = 0
btnMenu = null
menuOn = false
var myFrame = "<iframe id='Hfrm'
style='visibility:hidden;position:absolute;border-style:hidden;border:none;top:0px;left:0px;width:0px;height:0px;z-index:98;filter:alpha(opacity=80,
finishopacity=300, style=1, startx=0, starty=0, finishx=140,
finishy=270);-moz-opacity:.9;'>asdasds</iframe>"
document.write(myFrame)
if ((is.mac) && (is.ie))
{
document.getElementById("ddimagetabs").style.cssFloat="both";
}
prev = 0
function chgBgk(e)
{
e.parentNode.style.background="#669933"
if ((prev != 0) && (prev != e)) {
prev.parentNode.style.background="#E3E7DB"
}
prev = e
}
function chgBgk1(e)
{
prev.parentNode.style.background="#E3E7DB"
}
</script>
<script type="text/javascript" language="JavaScript">
<!-- //
function GoToSearchResults(objForm)
{
var type = objForm.searchType.value;
var q = objForm.q.value;
var zip = objForm.zip.value;
if ( q == "" && zip == "" )
{
if (type == "people")
{
dcsMultiTrack('DCSext.click','1','DCSext.url','http://my.monster.com/NetworkingSearch/Search.aspx','DCSext.id','s1','gDcsId','DCS000007_2Q9M');
window.location =
"http://my.monster.com/NetworkingSearch/SearchResults.aspx?searchType=Keyword"
}
else
{
dcsMultiTrack('DCSext.click','1','DCSext.url','http://jobsearch.monster.com/index.asp','DCSext.id','s2','gDcsId','DCS000007_2Q9M');
window.location = "http://jobsearch.monster.com/index.asp"
}
}
else
{
if (q != "")
{
q = "&q=" + encodeURIComponent(q);
}
if (zip != "")
{
if(!zip.match(/\d{5}/))
{
alert("Please enter a valid ZIP code.");
return false;
}
if (type == "people")
{
zip = zip + "-50"
zip = "&pcrad=" + encodeURIComponent(zip);
}
else
{
zip = "&zip=" + encodeURIComponent(zip);
}
}
if (type == "people")
{
dcsMultiTrack('DCSext.click','1','DCSext.url','http://my.monster.com/NetworkingSearch/Search.aspx','DCSext.id','s3','gDcsId','DCS000007_2Q9M');
window.location =
"http://my.monster.com/NetworkingSearch/SearchResults.aspx?opt=go&searchType=Keyword"
+ q + zip;
}
else
{
dcsMultiTrack('DCSext.click','1','DCSext.url','http://jobsearch.monster.com/jobsearch.asp','DCSext.id','s4','gDcsId','DCS000007_2Q9M');
window.location =
"http://jobsearch.monster.com/jobsearch.asp?opt=go&sort=rv&vw=b&cy=US&re=14&brd=1%2C1862%2C1863&rad=50"
+ q + zip;
}
}
return false;
}
// -->
</script>
<!-- START OF SmartSource Data Collector TAG -->
<!-- WebTrends SmartSource Data Collector -->
<!-- © 1996-2004 NetIQ Corporation. All rights reserved. -->
<!-- DateTime: 2004/05/03 13:43:43 -->
<SCRIPT LANGUAGE="Javascript"><!--
gVersion="1.0";
//-->
</SCRIPT>
<SCRIPT LANGUAGE="Javascript1.1"><!--
gVersion="1.1";
//-->
</SCRIPT>
<SCRIPT LANGUAGE="Javascript1.2"><!--
gVersion="1.2";
var RE={"%09":/\t/g,"%20":/
/g,"%23":/\#/g,"%26":/\&/g,"%2B":/\+/g,"%3F":/\?/g,"%5C":/\\/g};
//-->
</SCRIPT>
<SCRIPT LANGUAGE="Javascript1.3"><!--
gVersion="1.3";
//-->
</SCRIPT>
<SCRIPT LANGUAGE="Javascript1.4"><!--
gVersion="1.4";
//-->
</SCRIPT>
<SCRIPT LANGUAGE="Javascript1.5"><!--
gVersion="1.5";
//-->
</SCRIPT>
<SCRIPT LANGUAGE="JavaScript"><!--
var gImages=new Array;
var gIndex=0;
var DCS=new Object();
var WT=new Object();
var DCSext=new Object();
var gDomain="cookie.monster.com";
var gDcsId="DCS000003_6D4Q";
// WebTrends SmartSource Data Collector
// © 1996-2004 NetIQ Corporation. All rights reserved.
// DateTime: 2004/05/03 13:43:43
function dcsMultiTrack(){
for (var i=0;i<arguments.length;i++){
if (arguments[i].indexOf('WT.')==0){
WT[arguments[i].substring(3)]=arguments[i+1];
i++;
}
if (arguments[i].indexOf('DCS.')==0){
DCS[arguments[i].substring(4)]=arguments[i+1];
i++;
}
if (arguments[i].indexOf('DCSext.')==0){
DCSext[arguments[i].substring(7)]=arguments[i+1];
i++;
}
if (arguments[i]=='gDcsId'){
gDcsId=arguments[i+1];
i++;
}
}
var dCurrent=new Date();
DCS.dcsdat=dCurrent.getTime();
dcsTag();
}
function dcsPrintVariables()
{
var tagVariables="\nDomain = "+gDomain;
tagVariables+="\nDCSId = "+gDcsId;
for (N in DCS){
tagVariables+="\nDCS."+N+" = "+DCS[N];
}
for (N in WT){
tagVariables+="\nWT."+N+" = "+WT[N];
}
for (N in DCSext){
tagVariables+="\nDCSext."+N+" = "+DCSext[N];
}
window.alert(tagVariables);
}
function dcsVar(){
var dCurrent=new Date();
WT.tz=dCurrent.getTimezoneOffset()/60*-1;
if (WT.tz==0){
WT.tz="0";
}
WT.bh=dCurrent.getHours();
WT.ul=navigator.appName=="Netscape"?navigator.language:navigator.userLanguage;
if (typeof(screen)=="object"){
WT.cd=navigator.appName=="Netscape"?screen.pixelDepth:screen.colorDepth;
WT.sr=screen.width+"x"+screen.height;
}
if (typeof(navigator.javaEnabled())=="boolean"){
WT.jo=navigator.javaEnabled()?"Yes":"No";
}
if (document.title){
WT.ti=document.title;
}
WT.js="Yes";
if (typeof(gVersion)!="undefined"){
WT.jv=gVersion;
}
DCS.dcsdat=dCurrent.getTime();
DCS.dcssip="www.monster.com";
DCS.dcsuri=window.location.pathname;
if (window.location.search){
DCS.dcsqry=window.location.search;
}
if ((window.document.referrer!="")&&(window.document.referrer!="-")){
if (!(navigator.appName=="Microsoft Internet
Explorer"&&parseInt(navigator.appVersion)<4)){
DCS.dcsref=window.document.referrer;
}
}
}
function A(N,V){
return "&"+N+"="+dcsEscape(V);
}
function dcsEscape(S){
if (typeof(RE)!="undefined"){
var retStr = new String(S);
for (R in RE){
retStr = retStr.replace(RE[R],R);
}
return retStr;
}
else{
return escape(S);
}
}
function dcsCreateImage(dcsSrc){
// window.alert(dcsSrc);
if (document.images){
gImages[gIndex]=new Image;
gImages[gIndex].src=dcsSrc;
gIndex++;
}
else{
document.write('<IMG BORDER="0" NAME="DCSIMG" WIDTH="1" HEIGHT="1"
SRC="'+dcsSrc+'">');
}
}
function dcsMeta(){
var myDocumentElements;
if (document.all){
myDocumentElements=document.all.tags("meta");
}
else if (document.documentElement){
myDocumentElements=document.getElementsByTagName("meta");
}
if (typeof(myDocumentElements)!="undefined"){
for (var i=1;i<=myDocumentElements.length;i++){
myMeta=myDocumentElements.item(i-1);
if (myMeta.name){
if (myMeta.name.indexOf('WT.')==0){
WT[myMeta.name.substring(3)]=myMeta.content;
}
else if (myMeta.name.indexOf('DCSext.')==0){
DCSext[myMeta.name.substring(7)]=myMeta.content;
}
else if (myMeta.name.indexOf('DCS.')==0){
DCS[myMeta.name.substring(4)]=myMeta.content;
}
}
}
}
}
function dcsAddOnClick(){
var FuncBody={
get:function(funcPtr){
var str=funcPtr.toString();
str=str.replace(/[^{]+/,"");
str=str.substring(0,str.length-1);
return str;
}
};
var clicktrack={
onclick:function(){
dcsMultiTrack('DCSext.click','1','DCSext.url',this.href,'DCSext.id',this.id,'DCSext.name',this.name,'gDcsId','DCS000007_2Q9M');
return true;
}
};
var submittrack={
onsubmit:function(){
dcsMultiTrack('DCSext.click','1','DCSext.url',this.action,'DCSext.id',this.id,'DCSext.name',this.name,'gDcsId','DCS000007_2Q9M');
return true;
}
};
var myDocumentElements;
if (document.all){
myDocumentElements=document.all.tags("a");
}
else if (document.documentElement){
myDocumentElements=document.getElementsByTagName("a");
}
if (typeof(myDocumentElements)!="undefined"){
for (var i=1;i<=myDocumentElements.length;i++){
myAnchor=myDocumentElements.item(i-1);
if (myAnchor.href){
if ( myAnchor.onclick != null ) {
myAnchor.onclick = new
Function(FuncBody.get(myAnchor.onclick)+";"+FuncBody.get(clicktrack.onclick));
}
else {
myAnchor.onclick = clicktrack.onclick;
}
}
}
}
if (document.all){
myDocumentElements=document.all.tags("area");
}
else if (document.documentElement){
myDocumentElements=document.getElementsByTagName("area");
}
if (typeof(myDocumentElements)!="undefined"){
for (var i=1;i<=myDocumentElements.length;i++){
myAnchor=myDocumentElements.item(i-1);
if (myAnchor.href){
if ( myAnchor.onclick != null ) {
myAnchor.onclick = new
Function(FuncBody.get(myAnchor.onclick)+";"+FuncBody.get(clicktrack.onclick));
}
else {
myAnchor.onclick = clicktrack.onclick;
}
}
}
}
if (document.all){
myDocumentElements=document.all.tags("form");
}
else if (document.documentElement){
myDocumentElements=document.getElementsByTagName("form");
}
if (typeof(myDocumentElements)!="undefined"){
for (var i=1;i<=myDocumentElements.length;i++){
myForm=myDocumentElements.item(i-1);
if (myForm.action){
if ( myForm.onsubmit != null ) {
myForm.onsubmit = new
Function(FuncBody.get(myForm.onsubmit)+";"+FuncBody.get(submittrack.onsubmit));
}
else {
myForm.onsubmit = submittrack.onsubmit;
}
}
}
}
}
function dcsTag(){
var
P="http"+(window.location.protocol.indexOf('https:')==0?'s':'')+"://"+gDomain+(gDcsId==""?'':'/'+gDcsId)+"/dcs.gif?";
for (N in DCS){
if (DCS[N]) {
P+=A(N,DCS[N]);
}
}
for (N in WT){
if (WT[N]) {
P+=A("WT."+N,WT[N]);
}
}
for (N in DCSext){
if (DCSext[N]) {
P+=A(N,DCSext[N]);
}
}
if (P.length>2048&&navigator.userAgent.indexOf('MSIE')>=0){
P=P.substring(0,2040)+"&WT.tu=1";
}
dcsCreateImage(P);
}
dcsVar();
dcsMeta();
//dcsPrintVariables();
dcsTag();
dcsAddOnClick();
//-->
</SCRIPT>
<!-- END OF Data Collection Server TAG -->
<!-- Start of DoubleClick Spotlight Tag: Please do not remove-->
<!-- Activity Name for this tag is:Homepage -->
<!-- Web site URL where tag should be placed: http://www.monster.com/ -->
<!-- Creation Date:8/28/2003 -->
<SCRIPT language="JavaScript">
var axel = Math.random()+"";
var a = axel * 10000000000000;
document.write('<IMG
SRC="http://ad.doubleclick.net/activity;src=870983;type=consu801;cat=homep810;ord='+
a + '?" WIDTH=1 HEIGHT=1 BORDER=0>');
</SCRIPT>
<NOSCRIPT>
<IMG
SRC="http://ad.doubleclick.net/activity;src=870983;type=consu801;cat=homep810;ord=1?"
WIDTH=1 HEIGHT=1 BORDER=0>
</NOSCRIPT>
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
thanks again.
Received on 2005-07-14