OpenSolaris rsync install script

Posted by 4Aiur on 2010/12/21 in OpenSolaris | ∞

#!/bin/bash
# OpenSolaris rsync installer
# Created by 4Aiur on 2010-12-21.

# define function
check_rsync_config () {
    if [ ! -d /opt/local/etc/rsync/ ]; then
        mkdir -p /opt/local/etc/rsync/
    fi
    
    if [ ! -f /opt/local/etc/rsync/rsyncd.conf ]; then
        cat > /opt/local/etc/rsync/rsyncd.conf <<\EOF
#maximum allowed connections
max connections = 10
#where to log
log file = /var/log/rsync.log
timeout = 300

[update]
comment = update downloads
path = /home/foo/rsync_data/
read only = false
list = yes
uid = foo
gid = foo 
hosts allow = 192.168.1.0/24
secrets file = /opt/local/etc/rsync/rsyncd.secrets
auth users = foo #enter username specified in secrets file
EOF
    fi
    if [ ! -f /opt/local/etc/rsync/rsyncd.secrets ]; then
        cat > /opt/local/etc/rsync/rsyncd.secrets <<\EOF
rsync:rsync_pass
EOF
        chmod 600 /opt/local/etc/rsync/rsyncd.secrets
    fi
}

check_svc_rsync () {
    if [ ! -f /var/svc/manifest/network/rsync.xml ]; then
        cat > /var/svc/manifest/network/rsync.xml <<\EOF
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='rsync'>

  <service name='network/rsync' type='service' version='4'>

    <create_default_instance enabled='false'/>

    <single_instance/>

    <!--
    If there's no network, then there's no point in running 
    -->
    <dependency
      name='loopback'
      grouping='require_all'
      restart_on='error'
      type='service'>
      <service_fmri value='svc:/network/loopback:default'/>
    </dependency>

    <dependency
      name='physical'
      grouping='require_all'
      restart_on='error'
      type='service'>
      <service_fmri value='svc:/network/physical:default'/>
    </dependency>
    <dependency
      name='config_data'
      grouping='require_all'
      restart_on='restart'
      type='path'>
      <service_fmri value='file://localhost//opt/local/etc/rsync/rsyncd.conf'/>
    </dependency>
    <dependency
      name='fs-local'
      grouping='require_all'
      restart_on='none'
      type='service'>
      <service_fmri value='svc:/system/filesystem/local'/>
    </dependency>

    <exec_method
      type='method'
      name='start'
      exec='/opt/local/bin/rsync --daemon'
      timeout_seconds='60'/>

    <exec_method
      type='method'
      name='stop'
      exec=':kill'
      timeout_seconds='60'/>

    <exec_method
      type='method'
      name='refresh'
      exec=':kill -HUP'
      timeout_seconds='60'/>

    <stability value='Unstable'/>

    <template>
      <common_name>
        <loctext xml:lang='C'>RSYNC daemon</loctext>
      </common_name>

      <documentation>
        <manpage title='rsync' section='7'/>
        <doc_link name='rsync.org' uri='http://www.rsync.org/docs/'/>
      </documentation>
    </template>
  </service>
</service_bundle>
EOF
    fi
}

rotate_log () {
    logadm -w rsync -s 10m -C 10 -t '/var/log/rsync.log.%Y-%m' \
    -a '/usr/sbin/svcadm restart rsync' /var/log/rsync.log
}

# Main
echo "setup rsync server start"
# Check rsync package
if ! pkgin list | grep rsync >/dev/null; then
    pkgin -y in rsync
fi

# Check rsync server config file
check_rsync_config

# Check rsync service config
check_svc_rsync

# Import rsync.xml
if ! svcs -a | grep net-snmp >/dev/null; then
    svccfg import /var/svc/manifest/network/rsync.xml
fi
svcadm enable rsync

# Add log rotate config
rotate_log

echo "setup rsync server done"

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

#!/bin/bash

# OpenSolaris rsync installer

# Created by 4Aiur on 2010-12-21.

# define function

check_rsync_config () {

if [ ! -d /opt/local/etc/rsync/ ]; then

mkdir -p /opt/local/etc/rsync/

if [ ! -f /opt/local/etc/rsync/rsyncd.conf ]; then

cat > /opt/local/etc/rsync/rsyncd.conf <<\EOF

#maximum allowed connections

max connections = 10

#where to log

log file = /var/log/rsync.log

timeout = 300

[update]

comment = update downloads

path = /home/foo/rsync_data/

read only = false

list = yes

uid = foo

gid = foo

hosts allow = 192.168.1.0/24

secrets file = /opt/local/etc/rsync/rsyncd.secrets

auth users = foo #enter username specified in secrets file

EOF

if [ ! -f /opt/local/etc/rsync/rsyncd.secrets ]; then

cat > /opt/local/etc/rsync/rsyncd.secrets <<\EOF

rsync:rsync_pass

EOF

chmod 600 /opt/local/etc/rsync/rsyncd.secrets

}

check_svc_rsync () {

if [ ! -f /var/svc/manifest/network/rsync.xml ]; then

cat > /var/svc/manifest/network/rsync.xml <<\EOF

<?xml version="1.0"?>

<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<service_bundle type='manifest' name='rsync'>

<create_default_instance enabled='false'/>

<single_instance/>

<!--

If there's no network, then there's no point in running

-->

<dependency

name='loopback'

grouping='require_all'

restart_on='error'

type='service'>

<service_fmri value='svc:/network/loopback:default'/>

</dependency>

<dependency

name='physical'

grouping='require_all'

restart_on='error'

type='service'>

<service_fmri value='svc:/network/physical:default'/>

</dependency>

<dependency

name='config_data'

grouping='require_all'

restart_on='restart'

type='path'>

<service_fmri value='file://localhost//opt/local/etc/rsync/rsyncd.conf'/>

</dependency>

<dependency

name='fs-local'

grouping='require_all'

restart_on='none'

type='service'>

<service_fmri value='svc:/system/filesystem/local'/>

</dependency>

<exec_method

type='method'

name='start'

exec='/opt/local/bin/rsync --daemon'

timeout_seconds='60'/>

<exec_method

type='method'

name='stop'

exec=':kill'

timeout_seconds='60'/>

<exec_method

type='method'

name='refresh'

exec=':kill -HUP'

timeout_seconds='60'/>

<common_name>

<loctext xml:lang='C'>RSYNC daemon</loctext>

</common_name>

<doc_link name='rsync.org' uri='http://www.rsync.org/docs/'/>

</documentation>

</template>

</service>

</service_bundle>

EOF

}

rotate_log () {

logadm -w rsync -s 10m -C 10 -t '/var/log/rsync.log.%Y-%m' \

-a '/usr/sbin/svcadm restart rsync' /var/log/rsync.log

}

# Main

echo "setup rsync server start"

# Check rsync package

if ! pkgin list | grep rsync >/dev/null; then

pkgin -y in rsync

# Check rsync server config file

check_rsync_config

# Check rsync service config

check_svc_rsync

# Import rsync.xml

if ! svcs -a | grep net-snmp >/dev/null; then

svccfg import /var/svc/manifest/network/rsync.xml

svcadm enable rsync

# Add log rotate config

rotate_log

echo "setup rsync server done"

Tags: OpenSolaris, rsync

Logrotate on OpenSolaris

Posted by 4Aiur on 2010/12/21 in OpenSolaris | ∞

Logrotate on OpenSolaris On OpenSolaris, you use the logadm command, with the actual rotation being specified in /etc/logadm.conf

logadm -v -w rsync -s 10m -C 10 \
-t '/var/log/rsync.log.%Y-%m' \
-a '/usr/sbin/svcadm restart rsync' \
/var/log/rsync.log

logadm -v -w rsync -s 10m -C 10 \

-t '/var/log/rsync.log.%Y-%m' \

-a '/usr/sbin/svcadm restart rsync' \

/var/log/rsync.log

Testing with logadm -p now rsync seems to work just fine.

Tags: logrotate, OpenSolaris

OpenSolaris net-snmp install script

Posted by 4Aiur on 2010/12/16 in OpenSolaris | ∞

OpenSolaris net-snmp install script

#!/bin/bash
# OpenSolaris net-snmp installer
# Created by 4Aiur on 2010-12-14.

# define function
check_netsnmp_config () {
    if [ ! -f /var/net-snmp/snmpd.local.conf ]; then
        cat > /var/net-snmp/snmpd.local.conf <<\EOF
###############################################################################
# Access Control
###############################################################################
####
# First, map the community name (COMMUNITY) into a security name
# (local and mynetwork, depending on where the request is coming
# from):
#       sec.name  source          community
com2sec local     localhost       public
com2sec mynetwork NETWORK/24      COMMUNITY

####
# Second, map the security names into group names:
#               sec.model  sec.name
group MyRWGroup v2c        local
group MyRWGroup usm        local
group MyROGroup v2c        mynetwork
group MyROGroup usm        mynetwork

####
# Third, create a view for us to let the groups have rights to:
#           incl/excl subtree                          mask
view all    included  .1.3.6.1.4.1.2021
view all    included  .1.3.6.1.2.1.1
view all    included  .1.3.6.1.2.1.2.2
view all    included  .1.3.6.1.2.1.25.1.1               

####
# Finally, grant the 2 groups access to the 1 view with different
# write permissions:
#                context sec.model sec.level match  read   write  notif
access MyROGroup ""      any       noauth    exact  all    none   none
access MyRWGroup ""      any       noauth    exact  all    all    none

###############################################################################
# System contact information
syslocation Right here, right now.
syscontact Me <me@somewhere.org>

###############################################################################
# Process checks.
# Make sure sshd is running
proc sshd

###############################################################################
# disk checks
# disk PATH [MIN=DEFDISKMINIMUMSPACE]
# Check the / partition and make sure it contains at least 20 percent.
disk / 20%

###############################################################################
# load average checks
# load [1MAX=DEFMAXLOADAVE] [5MAX=DEFMAXLOADAVE] [15MAX=DEFMAXLOADAVE]
load 12 14 14

###############################################################################
# Executables/scripts
# exec NAME PROGRAM [ARGS ...]
exec echotest /bin/echo hello world
EOF
    fi
    chmod 600 /var/net-snmp/snmpd.local.conf
}

check_svc_netsnmp () {
    if [ ! -f /var/svc/manifest/application/management/net-snmp.xml ]; then
        cat > /var/svc/manifest/application/management/net-snmp.xml <<\EOF
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<!--
    CDDL HEADER START
   
    The contents of this file are subject to the terms of the
    Common Development and Distribution License (the "License").
    You may not use this file except in compliance with the License.
   
    You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    or http://www.opensolaris.org/os/licensing.
    See the License for the specific language governing permissions
    and limitations under the License.
   
    When distributing Covered Code, include this CDDL HEADER in each
    file and include the License file at usr/src/OPENSOLARIS.LICENSE.
    If applicable, add the following below this CDDL HEADER, with the
    fields enclosed by brackets "[]" replaced with your own identifying
    information: Portions Copyright [yyyy] [name of copyright owner]
   
    CDDL HEADER END
   
    Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
    Use is subject to license terms.

    ident    "@(#)net-snmp.xml    1.1    09/07/06 SMI"

    NOTE:  This service description is not editable; its contents
    may be overwritten by package or patch operations, including
    operating system upgrade.  Make customizations in a different
    file.

    Service manifest for the net-snmp daemon
-->

<service_bundle type='manifest' name='SUNWnet-snmp-core:net-snmp'>

<service
    name='application/management/net-snmp'
    type='service'
    version='1'>

    <create_default_instance enabled='false' />

    <single_instance />

    <dependency
        name='milestone'
        grouping='require_all'
        restart_on='none'
        type='service'> 
        <service_fmri value='svc:/milestone/sysconfig' />
    </dependency>

    <!-- Need / & /usr filesystems mounted, /var mounted read/write -->
    <dependency
        name='fs-local'
        type='service'
        grouping='require_all'
        restart_on='none'>
            <service_fmri value='svc:/system/filesystem/local' />
    </dependency>

    <dependency
        name='name-services'
        grouping='optional_all'
        restart_on='none'
        type='service'>
        <service_fmri value='svc:/milestone/name-services' />
    </dependency>

    <dependency
        name='system-log'
        grouping='optional_all'
        restart_on='none'
        type='service'>
        <service_fmri value='svc:/system/system-log' />
    </dependency>

    <dependency
        name='rstat'
        grouping='optional_all'
        restart_on='none'
        type='service'>
        <service_fmri value='svc:/network/rpc/rstat' />
    </dependency>

    <dependency name='cryptosvc'
        grouping='require_all'
        restart_on='restart'
        type='service'>
            <service_fmri value='svc:/system/cryptosvc' />
    </dependency>

    <dependency
        name='network'
        grouping='require_all'
        restart_on='restart'
        type='service'>
            <service_fmri value='svc:/milestone/network' />
    </dependency>

    <dependency
        name='config-file'
        grouping='require_all'
        restart_on='refresh'
        type='path'>
            <service_fmri 
               value='file://localhost/var/net-snmp/snmpd.local.conf' />
    </dependency>

    <exec_method
            type='method'
        name='start'
        exec='/lib/svc/method/svc-net-snmp'
        timeout_seconds='60'>
    </exec_method>

    <exec_method
       type='method'
       name='stop'
       exec=':kill'
       timeout_seconds='60'>
    </exec_method>

    <exec_method
       type='method'
       name='refresh'
       exec=':kill -HUP'
       timeout_seconds='60'>
    </exec_method>

    <property_group name='general' type='framework'>
        <!-- to start/stop net-snmp -->
        <propval name='action_authorization' type='astring'
            value='solaris.smf.manage.net-snmp' />
        <propval name='value_authorization' type='astring'
            value='solaris.smf.manage.net-snmp' />
        <propval name='arch_type' type='integer' value='0' />
    </property_group>
    
    <stability value='Unstable' />

    <template>
        <common_name>
            <loctext xml:lang='C'>
            net-snmp SNMP daemon
            </loctext>
        </common_name>

        <documentation>
            <manpage title='snmpd' section='8' 
                manpath='/usr/share/man/' />
        </documentation>

    </template>

</service>

</service_bundle>
EOF
    fi
}
# Check net-snmp package
if ! pkgin list | grep net-snmp >/dev/null; then
    pkgin -y in net-snmp
fi

# Check directory
if [ ! -d /var/net-snmp/ ]; then
    mkdir -p /var/net-snmp/
fi

# Check net-snmp config file.
check_netsnmp_config

# Check net-snmp service config
check_svc_netsnmp

# Import net-snmp.xml
if ! svcs -a | grep net-snmp >/dev/null; then
    svccfg import /var/svc/manifest/application/management/net-snmp.xml
fi
svcadm enable net-snmp
sleep 3
echo "Install net-snmp done."
echo "check net-snmp log."
tail /var/svc/log/application-management-net-snmp:default.log
echo "lookup net-snmp listen port"
netstat -an -P udp | grep 161

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

#!/bin/bash

# OpenSolaris net-snmp installer

# Created by 4Aiur on 2010-12-14.

# define function

check_netsnmp_config () {

if [ ! -f /var/net-snmp/snmpd.local.conf ]; then

cat > /var/net-snmp/snmpd.local.conf <<\EOF

###############################################################################

# Access Control

###############################################################################

####

# First, map the community name (COMMUNITY) into a security name

# (local and mynetwork, depending on where the request is coming

# from):

# sec.name source community

com2sec local localhost public

com2sec mynetwork NETWORK/24 COMMUNITY

####

# Second, map the security names into group names:

# sec.model sec.name

group MyRWGroup v2c local

group MyRWGroup usm local

group MyROGroup v2c mynetwork

group MyROGroup usm mynetwork

####

# Third, create a view for us to let the groups have rights to:

# incl/excl subtree mask

view all included .1.3.6.1.4.1.2021

view all included .1.3.6.1.2.1.1

view all included .1.3.6.1.2.1.2.2

view all included .1.3.6.1.2.1.25.1.1

####

# Finally, grant the 2 groups access to the 1 view with different

# write permissions:

# context sec.model sec.level match read write notif

access MyROGroup "" any noauth exact all none none

access MyRWGroup "" any noauth exact all all none

###############################################################################

# System contact information

syslocation Right here, right now.

syscontact Me <me@somewhere.org>

###############################################################################

# Process checks.

# Make sure sshd is running

proc sshd

###############################################################################

# disk checks

# disk PATH [MIN=DEFDISKMINIMUMSPACE]

# Check the / partition and make sure it contains at least 20 percent.

disk / 20%

###############################################################################

# load average checks

# load [1MAX=DEFMAXLOADAVE] [5MAX=DEFMAXLOADAVE] [15MAX=DEFMAXLOADAVE]

load 12 14 14

###############################################################################

# Executables/scripts

# exec NAME PROGRAM [ARGS ...]

exec echotest /bin/echo hello world

EOF

chmod 600 /var/net-snmp/snmpd.local.conf

}

check_svc_netsnmp () {

if [ ! -f /var/svc/manifest/application/management/net-snmp.xml ]; then

cat > /var/svc/manifest/application/management/net-snmp.xml <<\EOF

<?xml version="1.0"?>

<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">

<!--

CDDL HEADER START

The contents of this file are subject to the terms of the

Common Development and Distribution License (the "License").

You may not use this file except in compliance with the License.

You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

or http://www.opensolaris.org/os/licensing.

See the License for the specific language governing permissions

and limitations under the License.

When distributing Covered Code, include this CDDL HEADER in each

file and include the License file at usr/src/OPENSOLARIS.LICENSE.

If applicable, add the following below this CDDL HEADER, with the

fields enclosed by brackets "[]" replaced with your own identifying

information: Portions Copyright [yyyy] [name of copyright owner]

CDDL HEADER END

Use is subject to license terms.

ident "@(#)net-snmp.xml 1.1 09/07/06 SMI"

NOTE: This service description is not editable; its contents

may be overwritten by package or patch operations, including

operating system upgrade. Make customizations in a different

file.

Service manifest for the net-snmp daemon

-->

<service_bundle type='manifest' name='SUNWnet-snmp-core:net-snmp'>

<service

name='application/management/net-snmp'

type='service'

version='1'>

<create_default_instance enabled='false' />

<single_instance />

<dependency

name='milestone'

grouping='require_all'

restart_on='none'

type='service'>

<service_fmri value='svc:/milestone/sysconfig' />

</dependency>

<dependency

name='fs-local'

type='service'

grouping='require_all'

restart_on='none'>

<service_fmri value='svc:/system/filesystem/local' />

</dependency>

<dependency

name='name-services'

grouping='optional_all'

restart_on='none'

type='service'>

<service_fmri value='svc:/milestone/name-services' />

</dependency>

<dependency

name='system-log'

grouping='optional_all'

restart_on='none'

type='service'>

<service_fmri value='svc:/system/system-log' />

</dependency>

<dependency

name='rstat'

grouping='optional_all'

restart_on='none'

type='service'>

<service_fmri value='svc:/network/rpc/rstat' />

</dependency>

<dependency name='cryptosvc'

grouping='require_all'

restart_on='restart'

type='service'>

<service_fmri value='svc:/system/cryptosvc' />

</dependency>

<dependency

name='network'

grouping='require_all'

restart_on='restart'

type='service'>

<service_fmri value='svc:/milestone/network' />

</dependency>

<dependency

name='config-file'

grouping='require_all'

restart_on='refresh'

type='path'>

<service_fmri

value='file://localhost/var/net-snmp/snmpd.local.conf' />

</dependency>

<exec_method

type='method'

name='start'

exec='/lib/svc/method/svc-net-snmp'

timeout_seconds='60'>

</exec_method>

<exec_method

type='method'

name='stop'

exec=':kill'

timeout_seconds='60'>

</exec_method>

<exec_method

type='method'

name='refresh'

exec=':kill -HUP'

timeout_seconds='60'>

</exec_method>

<property_group name='general' type='framework'>

<propval name='action_authorization' type='astring'

value='solaris.smf.manage.net-snmp' />

<propval name='value_authorization' type='astring'

value='solaris.smf.manage.net-snmp' />

</property_group>

<common_name>

net-snmp SNMP daemon

</loctext>

</common_name>

<manpage title='snmpd' section='8'

manpath='/usr/share/man/' />

</documentation>

</template>

</service>

</service_bundle>

EOF

}

# Check net-snmp package

if ! pkgin list | grep net-snmp >/dev/null; then

pkgin -y in net-snmp

# Check directory

if [ ! -d /var/net-snmp/ ]; then

mkdir -p /var/net-snmp/

# Check net-snmp config file.

check_netsnmp_config

# Check net-snmp service config

check_svc_netsnmp

# Import net-snmp.xml

if ! svcs -a | grep net-snmp >/dev/null; then

svccfg import /var/svc/manifest/application/management/net-snmp.xml

svcadm enable net-snmp

sleep 3

echo "Install net-snmp done."

echo "check net-snmp log."

tail /var/svc/log/application-management-net-snmp:default.log

echo "lookup net-snmp listen port"

netstat -an -P udp | grep 161

Tags: net-snmp, OpenSolaris

使用python获取mp3中的歌词

Posted by 4Aiur on 2010/12/09 in Python | ∞

使用python获取mp3中的歌词安装依赖库eyeD3

$ sudo port install py26-eyed3

1	$ sudo port install py26-eyed3

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
dump_mp3_lyris.py

Created by 4Aiur on 2010-12-09.
"""

import os
import eyeD3

def get_mp3_file(dir):
    mp3_files = []
    files = os.listdir(dir)
    for file in files:
        if os.path.splitext(file)[1] == '.mp3':
            mp3_files.append(os.path.join(dir,file))
    return mp3_files

def dump_lyrics(mp3_file):
    print('parse %s' % (mp3_file))
    lyrics_file = '%s.txt' % (os.path.splitext(mp3_file)[0])
    fp = open(lyrics_file, 'w')
    tag = eyeD3.Tag()
    tag.link(mp3_file)
    fp.write('Title: %s\n\n' % (tag.getTitle().encode('utf-8', 'ignore')))
    lyrics = tag.getLyrics()
    for item in lyrics:
        for line in item.lyrics.splitlines():
            result = line.encode('utf-8', 'ignore')
            fp.write( result + '\n')
    fp.close()
    print('write %s done.' % (lyrics_file))
    return

def main():
    mp3_files = get_mp3_file('/Users/4aiur/Shares/englishpod')
    for mp3_file in mp3_files:
        dump_lyrics(mp3_file)

if __name__ == '__main__':
    main()

#!/usr/bin/env python

# -*- coding: utf-8 -*-

"""

dump_mp3_lyris.py

Created by 4Aiur on 2010-12-09.

"""

import os

import eyeD3

def get_mp3_file(dir):

mp3_files = []

files = os.listdir(dir)

for file in files:

if os.path.splitext(file)[1] == '.mp3':

mp3_files.append(os.path.join(dir,file))

return mp3_files

def dump_lyrics(mp3_file):

print('parse %s' % (mp3_file))

lyrics_file = '%s.txt' % (os.path.splitext(mp3_file)[0])

fp = open(lyrics_file, 'w')

tag = eyeD3.Tag()

tag.link(mp3_file)

fp.write('Title: %s\n\n' % (tag.getTitle().encode('utf-8', 'ignore')))

lyrics = tag.getLyrics()

for item in lyrics:

for line in item.lyrics.splitlines():

result = line.encode('utf-8', 'ignore')

fp.write( result + '\n')

fp.close()

print('write %s done.' % (lyrics_file))

return

def main():

mp3_files = get_mp3_file('/Users/4aiur/Shares/englishpod')

for mp3_file in mp3_files:

dump_lyrics(mp3_file)

if __name__ == '__main__':

main()

Tags: Python

使用python下载存放在Index中的资源

Posted by 4Aiur on 2010/12/06 in Python | ∞

使用python下载存放在Index中的资源 Useing python threading download example. like this:

wget -P result/ -nd -nH -r -l 1 -np -A pdf --http-user=book --http-password=cubook "http://www.4aiur.net/book/Programing/Python/"

1	wget -P result/ -nd -nH -r -l 1 -np -A pdf --http-user=book --http-password=cubook "http://www.4aiur.net/book/Programing/Python/"

#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
thread_download.py
下载网页中某种后缀名的文件
Created by 4Aiur on 2010-12-06.
'''
__version__ = "$Revision: 0.90 $"

import threading
import httplib
import urlparse
import base64
import re
import os
from BeautifulSoup import BeautifulSoup
from Queue import Queue

download_queue = Queue()

def get_object_url(start_url, username, password, suffix):
    ''' 抓取并解析index文件，分析页面获取需要下载指定
    后缀名的元素 '''
    s = urlparse.urlparse(start_url)
    host = s.netloc
    path = s.path
    url_object = set()
    conn = httplib.HTTPConnection(host)
    params = None
    base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
    headers = {'User-agent': '4Aiur Crawler',
            'Authorization': 'Basic %s' % base64string}
    conn.request('GET', path, params, headers)
    response = conn.getresponse()
    content = response.read()
    content = unicode(content, errors="ignore")
    conn.close()
    soup = BeautifulSoup()
    soup.feed(content)
    items = soup.findAll('a')
    for item in items:
        href = item.get('href')
        if href.endswith(suffix):
            full_url = urlparse.urljoin(start_url, href)
            url_object.add(full_url)
    return url_object 

def download(n, save_path):
    '''download object'''
    while True:
        download_task = download_queue.get()
        host = download_task.host
        path = download_task.path
        username = download_task.username
        password = download_task.password
        re_separate = re.compile(r'/(?:/)*')
        file_name = re.split(re_separate, path)[-1]
        local_file = os.path.join(save_path, file_name)
        print('Thread-%d: downlaod %s to %s' % (n, download_task.url, local_file))
        params = None
        base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
        headers = {'User-agent': '4Aiur Crawler',
                'Authorization': 'Basic %s' % base64string}
        write_buffer = 104857600
        # 避免线程内代码出现异常，导致线程无法退出，把执行代码放入到异常判断中
        try:
            conn = httplib.HTTPConnection(host)
            conn.request('GET', path, params, headers)
            response = conn.getresponse()
            fp = open(local_file, 'wb')
            while True:
                block = response.read(write_buffer)
                if block:
                    fp.write(block)
                else:
                    break
            fp.close()
        finally:
            download_queue.task_done()

class DownloadObject(object):
    def __init__(self, url, username, password):
        self.url = url
        s = urlparse.urlparse(url)
        self.host = s.netloc
        self.path = s.path
        self.username = username
        self.password = password
        return

def main():
    num_threads = 3
    save_path = 'result'
    if not os.path.isdir(save_path):
        os.makedirs(save_path)
    # 开启下载线程
    for n in range(num_threads):
        t = threading.Thread(target=download, args=(n, save_path))
        t.setDaemon(True)
        t.start()

    start_url = 'http://www.4aiur.net/book/Programing/Python/'
    username = 'book'
    password = 'cubook'
    suffix = 'pdf'
    for url in get_object_url(start_url, username, password, suffix):
        download_queue.put(DownloadObject(url, username, password))
    # 等待线程结束
    download_queue.join()
    return

if __name__ == '__main__':
    main()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

#!/usr/bin/env python

# -*- coding: utf-8 -*-

'''

thread_download.py

下载网页中某种后缀名的文件

Created by 4Aiur on 2010-12-06.

'''

__version__ = "$Revision: 0.90 $"

import threading

import httplib

import urlparse

import base64

import re

import os

from BeautifulSoup import BeautifulSoup

from Queue import Queue

download_queue = Queue()

def get_object_url(start_url, username, password, suffix):

''' 抓取并解析index文件，分析页面获取需要下载指定

后缀名的元素 '''

s = urlparse.urlparse(start_url)

host = s.netloc

path = s.path

url_object = set()

conn = httplib.HTTPConnection(host)

params = None

base64string = base64.encodestring('%s:%s' % (username, password))[:-1]

headers = {'User-agent': '4Aiur Crawler',

'Authorization': 'Basic %s' % base64string}

conn.request('GET', path, params, headers)

response = conn.getresponse()

content = response.read()

content = unicode(content, errors="ignore")

conn.close()

soup = BeautifulSoup()

soup.feed(content)

items = soup.findAll('a')

for item in items:

href = item.get('href')

if href.endswith(suffix):

full_url = urlparse.urljoin(start_url, href)

url_object.add(full_url)

return url_object

def download(n, save_path):

'''download object'''

while True:

download_task = download_queue.get()

host = download_task.host

path = download_task.path

username = download_task.username

password = download_task.password

re_separate = re.compile(r'/(?:/)*')

file_name = re.split(re_separate, path)[-1]

local_file = os.path.join(save_path, file_name)

print('Thread-%d: downlaod %s to %s' % (n, download_task.url, local_file))

params = None

base64string = base64.encodestring('%s:%s' % (username, password))[:-1]

headers = {'User-agent': '4Aiur Crawler',

'Authorization': 'Basic %s' % base64string}

write_buffer = 104857600

# 避免线程内代码出现异常，导致线程无法退出，把执行代码放入到异常判断中

try:

conn = httplib.HTTPConnection(host)

conn.request('GET', path, params, headers)

response = conn.getresponse()

fp = open(local_file, 'wb')

while True:

block = response.read(write_buffer)

if block:

fp.write(block)

else:

break

fp.close()

finally:

download_queue.task_done()

class DownloadObject(object):

def __init__(self, url, username, password):

self.url = url

s = urlparse.urlparse(url)

self.host = s.netloc

self.path = s.path

self.username = username

self.password = password

return

def main():

num_threads = 3

save_path = 'result'

if not os.path.isdir(save_path):

os.makedirs(save_path)

# 开启下载线程

for n in range(num_threads):

t = threading.Thread(target=download, args=(n, save_path))

t.setDaemon(True)

t.start()

start_url = 'http://www.4aiur.net/book/Programing/Python/'

username = 'book'

password = 'cubook'

suffix = 'pdf'

for url in get_object_url(start_url, username, password, suffix):

download_queue.put(DownloadObject(url, username, password))

# 等待线程结束

download_queue.join()

return

if __name__ == '__main__':

main()

Tags: Python

Python 三元表达式

Posted by 4Aiur on 2010/12/06 in Python | ∞

Python 三元表达式

In [1]: x = 100 if True else 200

In [2]: x
Out[2]: 100

In [3]: x = 100 if False else 200

In [4]: x
Out[4]: 200

In [1]: x = 100 if True else 200

In [2]: x

Out[2]: 100

In [3]: x = 100 if False else 200

In [4]: x

Out[4]: 200

Tags: Python

4Aiur

OpenSolaris rsync install script

Logrotate on OpenSolaris

OpenSolaris net-snmp install script

使用python获取mp3中的歌词

使用python下载存放在Index中的资源

Python 三元表达式

Categories

Views

Archives

Meta