JMX monitoring
JMX monitoring
dataflow
java app -> jmxagent ->
jmxtrans -> jmx_output.log -> alert.sh -> send mail
java app -> jmxagent ->
Configuration jmx agent with authentication
JMX dynamically allocated random port, and it will bind the port at internal address.
If you connecting jmx through firewall or your servers on Amazon EC2,
maybe can’t connect to the jmx agent, So need to do some prepare.
add jmx agent parameter in your java startup script
cmd[0]='curl -s http://ifconfig.me/ip 2>/dev/null | tr -d "\n"'
cmd[1]='curl -s http://sputnick-area.net/ip 2>/dev/null'
for ((x=0; x
configuration jmx authentication
cat > /opt/app/conf/jmxremote.access /opt/app/conf/jmxremote.password
build JMXAgent.class to jmx-agent.jar
package example.rmi.agent;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.rmi.registry.LocateRegistry;
import java.util.HashMap;
import javax.management.MBeanServer;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXConnectorServerFactory;
import javax.management.remote.JMXServiceURL;
/**
* This class is used for the resolve the
* "Connecting Through Firewall Using JMX" issue.
*
* http://blogs.oracle.com/jmxetc/entry/connecting_through_firewall_using_jmx
*
* @author root
*
*/
public class JMXAgent {
private static int _rmiRegistryPort=3000;
public static void premain(String agentArgs) throws IOException {
// Ensure cryptographically strong random number generator used
// to choose the object number - see java.rmi.server.ObjID
//
System.setProperty("java.rmi.server.randomIDs", "true");
// Start an RMI registry on port specified by example.rmi.agent.port
// (default 3000).
//
final int port = Integer.parseInt(System.getProperty(
"yottaa.rmi.agent.port", String.valueOf(_rmiRegistryPort)));
System.out.println("Create RMI registry on port " + port);
LocateRegistry.createRegistry(port);
// Retrieve the PlatformMBeanServer.
//
System.out.println("Get the platform's MBean server");
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
// Environment map.
//
System.out.println("Initialize the environment map");
HashMap env = new HashMap();
// This where we would enable security - left out of this
// for the sake of the example....
//
// Create an RMI connector server.
//
// As specified in the JMXServiceURL the RMIServer stub will be
// registered in the RMI registry running in the local host on
// port 3000 with the name "jmxrmi". This is the same name the
// out-of-the-box management agent uses to register the RMIServer
// stub too.
//
// The port specified in "service:jmx:rmi://"+hostname+":"+port
// is the second port, where RMI connection objects will be exported.
// Here we use the same port as that we choose for the RMI registry.
// The port for the RMI registry is specified in the second part
// of the URL, in "rmi://"+hostname+":"+port
//
System.out.println("Create an RMI connector server");
final String hostname = InetAddress.getLocalHost().getHostName();
JMXServiceURL url = new JMXServiceURL("service:jmx:rmi://" + hostname
+ ":" + port + "/jndi/rmi://" + hostname + ":" + port
+ "/jmxrmi");
// Now create the server from the JMXServiceURL
//
JMXConnectorServer cs = JMXConnectorServerFactory.newJMXConnectorServer(url, env, mbs);
// Start the RMI connector server.
//
System.out.println("Start the RMI connector server on port " + port);
cs.start();
}
}
Install jmxtrans on your alert server
rpm -Uvh http://jmxtrans.googlecode.com/files/jmxtrans-250-0.noarch.rpm
ln -s /usr/java/jdk1.6.0_27/bin/jps /usr/bin/jps
Configuration jmxtrans
cat > /var/lib/jmxtrans/monitoring.json
Running Jmx Transformer
service jmxtrans start
# or run jmxtrans manually
cd /usr/share/jmxtrans/
/usr/share/jmxtrans/jmxtrans.sh start /var/lib/jmxtrans/monitoring.json
/usr/share/jmxtrans/jmxtrans.sh stop /var/lib/jmxtrans/monitoring.json
View jmxtrans log
cd /var/log/jmxtrans
tailf jmxtrans.log
Alert script thresholds config
cat > thresholds.conf
Alert script
cat > /opt/alert.sh /dev/null | head -${max_line} | \
awk -v Now=$now -v Expire=$expire '
BEGIN{
# configuration
# #parttern expression value
# HeapMemoryUsage_used Maximum 2500000000
# HeapMemoryUsage_committed Minimum 6000000000
while (getline = Values[Parttern]) {
return Hit=1
}
} else if (Expressions[Parttern] == "Minimum") {
if (Value = Now) {
scanner(Attr, Value)
if (Hit == 1) {
print
exit
}
} else {
exit
}
}
}
')
if [[ ! -z $message ]]; then
send_mail "$message"
fi
logger "done."
EOF
chmod +x /opt/alert.sh
Add script into crontab
crontab -l > tmp.cron
echo "* * * * * /opt/alert.sh >> /var/log/jmxtrans/alert.log 2>&1" >> tmp.cron
crontab tmp.cron
rm -f tmp.cron