flink源码分析-获取JVM最大堆内存

flink版本: flink-1.11.2

代码位置: org.apache.flink.runtime.util.EnvironmentInformation#getMaxJvmHeapMemory

如果设置了-Xmx参数,就返回这个参数,如果没设置就返回机器物理内存的1/4. 这里主要看各个机器内存的获取方法。

java 复制代码
	/**
	 * The maximum JVM heap size, in bytes.
	 *
	 * <p>This method uses the <i>-Xmx</i> value of the JVM, if set. If not set, it returns (as
	 * a heuristic) 1/4th of the physical memory size.
	 *
	 * @return The maximum JVM heap size, in bytes.
	 */
	public static long getMaxJvmHeapMemory() {
		final long maxMemory = Runtime.getRuntime().maxMemory();
		if(maxMemory != Long.MAX_VALUE) {
			// we have the proper max memory
			return maxMemory;
		} else {
			// max JVM heap size is not set - use the heuristic to use 1/4th of the physical memory
			final long physicalMemory = Hardware.getSizeOfPhysicalMemory();
			if(physicalMemory != -1) {
				// got proper value for physical memory
				return physicalMemory / 4;
			} else {
				throw new RuntimeException(
					"Could not determine the amount of free memory.\n" + "Please set the maximum memory for the JVM, e.g. -Xmx512M for 512 megabytes.");
			}
		}
	}

进入getSizeOfPhysicalMemory()方法,里面有获取各种操作系统物理内存的方法:

java 复制代码
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.flink.util.OperatingSystem;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Convenience class to extract hardware specifics of the computer executing the running JVM.
 */
public class Hardware {

	private static final Logger LOG = LoggerFactory.getLogger(Hardware.class);

	private static final String LINUX_MEMORY_INFO_PATH = "/proc/meminfo";

	private static final Pattern LINUX_MEMORY_REGEX = Pattern.compile("^MemTotal:\\s*(\\d+)\\s+kB$");

	// ------------------------------------------------------------------------
	
	/**
	 * Gets the number of CPU cores (hardware contexts) that the JVM has access to.
	 * 
	 * @return The number of CPU cores.
	 */
	public static int getNumberCPUCores() {

		// TODO_MA 注释: 获取 Cpu Cores
		return Runtime.getRuntime().availableProcessors();
	}

	/**
	 * Returns the size of the physical memory in bytes.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined.
	 */
	public static long getSizeOfPhysicalMemory() {
		// first try if the JVM can directly tell us what the system memory is
		// this works only on Oracle JVMs
		try {
			Class<?> clazz = Class.forName("com.sun.management.OperatingSystemMXBean");
			Method method = clazz.getMethod("getTotalPhysicalMemorySize");
			OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean();

			// someone may install different beans, so we need to check whether the bean
			// is in fact the sun management bean
			if (clazz.isInstance(operatingSystemMXBean)) {
				return (Long) method.invoke(operatingSystemMXBean);
			}
		}
		catch (ClassNotFoundException e) {
			// this happens on non-Oracle JVMs, do nothing and use the alternative code paths
		}
		catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
			LOG.warn("Access to physical memory size: " +
					"com.sun.management.OperatingSystemMXBean incompatibly changed.", e);
		}

		// we now try the OS specific access paths
		switch (OperatingSystem.getCurrentOperatingSystem()) {
			case LINUX:
				return getSizeOfPhysicalMemoryForLinux();

			case WINDOWS:
				return getSizeOfPhysicalMemoryForWindows();

			case MAC_OS:
				return getSizeOfPhysicalMemoryForMac();

			case FREE_BSD:
				return getSizeOfPhysicalMemoryForFreeBSD();

			case UNKNOWN:
				LOG.error("Cannot determine size of physical memory for unknown operating system");
				return -1;

			default:
				LOG.error("Unrecognized OS: " + OperatingSystem.getCurrentOperatingSystem());
				return -1;
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on a Linux-based
	 * operating system.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForLinux() {
		try (BufferedReader lineReader = new BufferedReader(new FileReader(LINUX_MEMORY_INFO_PATH))) {
			String line;
			while ((line = lineReader.readLine()) != null) {
				Matcher matcher = LINUX_MEMORY_REGEX.matcher(line);
				if (matcher.matches()) {
					String totalMemory = matcher.group(1);
					return Long.parseLong(totalMemory) * 1024L; // Convert from kilobyte to byte
				}
			}
			// expected line did not come
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (NumberFormatException e) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo') ", t);
			return -1;
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on a Mac OS-based
	 * operating system
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForMac() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("sysctl hw.memsize");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line;
			while ((line = bi.readLine()) != null) {
				if (line.startsWith("hw.memsize")) {
					long memsize = Long.parseLong(line.split(":")[1].trim());
					bi.close();
					proc.destroy();
					return memsize;
				}
			}

		} catch (Throwable t) {
			LOG.error("Cannot determine physical memory of machine for MacOS host", t);
			return -1;
		} finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (IOException ignored) {}
			}
		}
		return -1;
	}

	/**
	 * Returns the size of the physical memory in bytes on FreeBSD.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForFreeBSD() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("sysctl hw.physmem");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line;
			while ((line = bi.readLine()) != null) {
				if (line.startsWith("hw.physmem")) {
					long memsize = Long.parseLong(line.split(":")[1].trim());
					bi.close();
					proc.destroy();
					return memsize;
				}
			}
			
			LOG.error("Cannot determine the size of the physical memory for FreeBSD host " +
					"(using 'sysctl hw.physmem').");
			return -1;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for FreeBSD host " +
					"(using 'sysctl hw.physmem')", t);
			return -1;
		}
		finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (IOException ignored) {}
			}
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on Windows.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForWindows() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("wmic memorychip get capacity");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line = bi.readLine();
			if (line == null) {
				return -1L;
			}

			if (!line.startsWith("Capacity")) {
				return -1L;
			}

			long sizeOfPhyiscalMemory = 0L;
			while ((line = bi.readLine()) != null) {
				if (line.isEmpty()) {
					continue;
				}

				line = line.replaceAll(" ", "");
				sizeOfPhyiscalMemory += Long.parseLong(line);
			}
			return sizeOfPhyiscalMemory;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for Windows host " +
					"(using 'wmic memorychip')", t);
			return -1L;
		}
		finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (Throwable ignored) {}
			}
		}
	}

	// --------------------------------------------------------------------------------------------

	private Hardware() {}
}

另外注意try catch的这种用法:

java 复制代码
		try (BufferedReader lineReader = new BufferedReader(new FileReader(LINUX_MEMORY_INFO_PATH))) {
			String line;
			while ((line = lineReader.readLine()) != null) {
				Matcher matcher = LINUX_MEMORY_REGEX.matcher(line);
				if (matcher.matches()) {
					String totalMemory = matcher.group(1);
					return Long.parseLong(totalMemory) * 1024L; // Convert from kilobyte to byte
				}
			}
			// expected line did not come
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (NumberFormatException e) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
相关推荐
G皮T3 小时前
【Elasticsearch】自定义评分检索
大数据·elasticsearch·搜索引擎·查询·检索·自定义评分·_score
微风粼粼6 小时前
程序员在线接单
java·jvm·后端·python·eclipse·tomcat·dubbo
搞笑的秀儿7 小时前
信息新技术
大数据·人工智能·物联网·云计算·区块链
SelectDB7 小时前
SelectDB 在 AWS Graviton ARM 架构下相比 x86 实现 36% 性价比提升
大数据·架构·aws
二二孚日7 小时前
自用华为ICT云赛道Big Data第五章知识点-Flume海量日志聚合
大数据·华为
二二孚日9 小时前
自用华为ICT云赛道Big Data第四章知识点-Flink流批一体分布式实时处理引擎
大数据·华为
掘金-我是哪吒9 小时前
分布式微服务系统架构第158集:JavaPlus技术文档平台日更-JVM基础知识
jvm·分布式·微服务·架构·系统架构
xufwind10 小时前
spark standlone 集群离线安装
大数据·分布式·spark
abigalexy10 小时前
深入JVM底层-内存分配算法
jvm
AI数据皮皮侠11 小时前
中国区域10m空间分辨率楼高数据集(全国/分省/分市/免费数据)
大数据·人工智能·机器学习·分类·业界资讯