Yarn 之 run job

src/main/java/org/apache/hadoop/mapreduce/Job.java

cpp 复制代码
  /**
   * Monitor a job and print status in real-time as progress is made and tasks 
   * fail.
   * @return true if the job succeeded
   * @throws IOException if communication to the JobTracker fails
   */
  public boolean monitorAndPrintJob() 
      throws IOException, InterruptedException {
    String lastReport = null;
    Job.TaskStatusFilter filter;
    Configuration clientConf = getConfiguration();
    filter = Job.getTaskOutputFilter(clientConf);
    JobID jobId = getJobID();
    LOG.info("Running job: " + jobId);
    int eventCounter = 0;
    boolean profiling = getProfileEnabled();
cpp 复制代码
  /** Key in mapred-*.xml that sets progMonitorPollIntervalMillis */
  public static final String PROGRESS_MONITOR_POLL_INTERVAL_KEY =
    "mapreduce.client.progressmonitor.pollinterval";
  /** Default progMonitorPollIntervalMillis is 1000 ms. */
  static final int DEFAULT_MONITOR_POLL_INTERVAL = 1000;
cpp 复制代码
while (!isComplete() || !reportedAfterCompletion) {
      if (isComplete()) {
        reportedAfterCompletion = true;
      } else {
        Thread.sleep(progMonitorPollIntervalMillis);
      }
      if (status.getState() == JobStatus.State.PREP) {
        continue;
      }      
      if (!reportedUberMode) {
        reportedUberMode = true;
        LOG.info("Job " + jobId + " running in uber mode : " + isUber());
      }      
      String report = 
        (" map " + StringUtils.formatPercent(mapProgress(), 0)+
            " reduce " + 
            StringUtils.formatPercent(reduceProgress(), 0));
      if (!report.equals(lastReport)) {
        LOG.info(report);
        lastReport = report;
      }

      TaskCompletionEvent[] events = 
        getTaskCompletionEvents(eventCounter, 10); 
      eventCounter += events.length;
      printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
    }
相关推荐
LinXunFeng5 小时前
Obsidian - 使用 Share Note 分享笔记并自部署
前端·笔记·github
乘风gg9 小时前
为什么AI 时代来临,大部分人吃不到红利
前端·ai编程·claude
恋猫de小郭10 小时前
Android 限制侧载新进展,谷歌联合国内厂商推验证计划
android·前端·flutter
IT_陈寒10 小时前
Redis内存爆了,原来我漏掉了这个致命配置
前端·人工智能·后端
恋猫de小郭10 小时前
解读 Android 17 全新内存限制,有没有“豁免”后门?
android·前端·flutter
小bo波10 小时前
从"任意文件复制"深挖Java I/O:字符流与字节流的本质抉择
java·nio·io流·后端开发·文件复制
Hyyy11 小时前
理解LLM的基本工作原理:预训练、微调、推理的区别
前端
Gatlin12 小时前
前端逆向与反逆向:一场猫鼠游戏的底层逻辑与实战
前端
Pedantic12 小时前
本地通知(Local Notifications)学习笔记
前端
森蓝情丶12 小时前
我给 AI 搭了个法庭:一个前端仔的 LangGraph 实战全记录
前端·后端