# HG changeset patch # Parent 6bd5c687f11af402307c27fbdec064d980f56a05 diff --git a/src/share/vm/gc_implementation/g1/concurrentMark.cpp b/src/share/vm/gc_implementation/g1/concurrentMark.cpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp @@ -489,7 +489,7 @@ _marking_task_overhead(1.0), _cleanup_sleep_factor(0.0), _cleanup_task_overhead(1.0), - _cleanup_list("Cleanup List"), + _cleanup_list("Cleanup List", false, true), _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), _card_bm((heap_rs.size() + CardTableModRefBS::card_size - 1) >> CardTableModRefBS::card_shift, @@ -889,7 +889,7 @@ class NoteStartOfMarkHRClosure: public HeapRegionClosure { public: bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { + if (r->has_own_objects()) { r->note_start_of_marking(); } return false; @@ -1369,7 +1369,7 @@ // to 1 the bits on the region bitmap that correspond to its // associated "continues humongous" regions. void set_bit_for_region(HeapRegion* hr) { - assert(!hr->continuesHumongous(), "should have filtered those out"); + assert(hr->has_own_objects(), "should have filtered those out"); BitMap::idx_t index = (BitMap::idx_t) hr->hrs_index(); if (!hr->startsHumongous()) { @@ -1405,7 +1405,7 @@ bool doHeapRegion(HeapRegion* hr) { - if (hr->continuesHumongous()) { + if (!hr->has_own_objects()) { // We will ignore these here and process them when their // associated "starts humongous" region is processed (see // set_bit_for_heap_region()). Note that we cannot rely on their @@ -1417,7 +1417,16 @@ } HeapWord* ntams = hr->next_top_at_mark_start(); - HeapWord* start = hr->bottom(); + HeapWord* start = hr->first_object_start(); + + if (ntams < start) { + // For humongous regions that we have allocated other objects into the ntams may have been set to + // _bottom before we started allocating objects into it. That means tath first_object_start may be + // higher than ntams. The code below can handle ntams == start, so as a workaround use that. + assert(hr->continuesHumongous(), "this must be a hum region we have allocated other objects into"); + ntams = start; + } + assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), err_msg("Preconditions not met - " @@ -1527,7 +1536,7 @@ int failures() const { return _failures; } bool doHeapRegion(HeapRegion* hr) { - if (hr->continuesHumongous()) { + if (!hr->has_own_objects()) { // We will ignore these here and process them when their // associated "starts humongous" region is processed (see // set_bit_for_heap_region()). Note that we cannot rely on their @@ -1701,8 +1710,7 @@ CMCountDataClosureBase(g1h, region_bm, card_bm) { } bool doHeapRegion(HeapRegion* hr) { - - if (hr->continuesHumongous()) { + if (!hr->has_own_objects()) { // We will ignore these here and process them when their // associated "starts humongous" region is processed (see // set_bit_for_heap_region()). Note that we cannot rely on their @@ -1834,7 +1842,7 @@ const HeapRegionSetCount& humongous_regions_removed() { return _humongous_regions_removed; } bool doHeapRegion(HeapRegion *hr) { - if (hr->continuesHumongous()) { + if (!hr->has_own_objects()) { return false; } // We use a claim value of zero here because all regions @@ -1848,10 +1856,36 @@ if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { _freed_bytes += hr->used(); hr->set_containing_set(NULL); - if (hr->isHumongous()) { - assert(hr->startsHumongous(), "we should only see starts humongous"); + if (hr->startsHumongous()) { + HeapWord* hum_top = hr->top(); _humongous_regions_removed.increment(1u, hr->capacity()); - _g1->free_humongous_region(hr, _local_cleanup_list, true); + HeapRegion* last_region = _g1->free_humongous_region(hr, _local_cleanup_list, true); + + if (last_region != NULL) { + // We claimed this region and now turned it into an old region. + // Need to do what we would have done to an old region. + assert(last_region->used() > 0, "Should not have kept this region"); + _g1->reset_gc_time_stamps(last_region); + last_region->note_end_of_marking(); + _max_live_bytes += last_region->max_live_bytes(); + if (last_region->used() > 0 && last_region->max_live_bytes() == 0) { + _freed_bytes += last_region->used(); + last_region->set_containing_set(NULL); + last_region->reset_non_humongous_space(); + last_region->set_first_object_start(last_region->bottom()); + _old_regions_removed.increment(1u, last_region->capacity()); + _g1->free_region(last_region, _local_cleanup_list, true); + if (G1VerboseAllocObjsInHCRegions) { + gclog_or_tty->print_cr("Freed up region %u.", last_region->hrs_index()); + } + } else { + last_region->make_old(); + last_region->rem_set()->do_cleanup_work(_hrrs_cleanup_task); + } + // We are counting the same memory twice in the last humongous region. Decuct that. + size_t used_overlap = byte_size(last_region->bottom(), hum_top); + _freed_bytes -= used_overlap; + } } else { _old_regions_removed.increment(1u, hr->capacity()); _g1->free_region(hr, _local_cleanup_list, true); @@ -1891,7 +1925,7 @@ void work(uint worker_id) { double start = os::elapsedTime(); - FreeRegionList local_cleanup_list("Local Cleanup List"); + FreeRegionList local_cleanup_list("Local Cleanup List", false, true); HRRSCleanupTask hrrs_cleanup_task; G1NoteEndOfConcMarkClosure g1_note_end(_g1h, worker_id, &local_cleanup_list, &hrrs_cleanup_task); @@ -2071,6 +2105,7 @@ } else { g1_par_note_end_task.work(0); } + g1h->check_partial(); g1h->check_gc_time_stamps(); if (!cleanup_list_is_empty()) { @@ -2146,7 +2181,7 @@ G1CollectedHeap* g1h = G1CollectedHeap::heap(); _cleanup_list.verify_list(); - FreeRegionList tmp_free_list("Tmp Free List"); + FreeRegionList tmp_free_list("Tmp Free List", false, true); if (G1ConcRegionFreeingVerbose) { gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " @@ -2657,7 +2692,6 @@ str = " O"; } else { HeapRegion* hr = _g1h->heap_region_containing(obj); - guarantee(hr != NULL, "invariant"); bool over_tams = _g1h->allocated_since_marking(obj, hr, _vo); bool marked = _g1h->is_marked(obj, _vo); @@ -2838,7 +2872,7 @@ // iterations) but it should not introduce and correctness issues. HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); HeapWord* bottom = curr_region->bottom(); - HeapWord* end = curr_region->end(); + HeapWord* end = bottom + HeapRegion::GrainWords; //curr_region->end(); HeapWord* limit = curr_region->next_top_at_mark_start(); if (verbose_low()) { @@ -2852,6 +2886,7 @@ HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); if (res == finger) { // we succeeded + assert(pointer_delta(end, finger) <= HeapRegion::GrainWords, "Moving too fast"); // notice that _finger == end cannot be guaranteed here since, // someone else might have moved the finger even further @@ -3042,7 +3077,7 @@ _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } bool doHeapRegion(HeapRegion* hr) { - if (hr->continuesHumongous()) { + if (!hr->has_own_objects()) { // We will ignore these here and process them when their // associated "starts humongous" region is processed. // Note that we cannot rely on their associated @@ -3419,10 +3454,9 @@ } void CMTask::setup_for_region(HeapRegion* hr) { - // Separated the asserts so that we know which one fires. assert(hr != NULL, "claim_region() should have filtered out continues humongous regions"); - assert(!hr->continuesHumongous(), + assert(hr->has_own_objects(), "claim_region() should have filtered out continues humongous regions"); if (_cm->verbose_low()) { @@ -3431,20 +3465,18 @@ } _curr_region = hr; - _finger = hr->bottom(); + _finger = hr->first_object_start(); update_region_limit(); } void CMTask::update_region_limit() { HeapRegion* hr = _curr_region; - HeapWord* bottom = hr->bottom(); - HeapWord* limit = hr->next_top_at_mark_start(); - - if (limit == bottom) { + HeapWord* bottom = hr->first_object_start(); + HeapWord* tams = hr->next_top_at_mark_start(); + + if (tams == bottom) { if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] found an empty region " - "["PTR_FORMAT", "PTR_FORMAT")", - _worker_id, bottom, limit); + gclog_or_tty->print_cr("[%u] found an empty region ["PTR_FORMAT", "PTR_FORMAT")", _worker_id, bottom, tams); } // The region was collected underneath our feet. // We set the finger to bottom to ensure that the bitmap @@ -3452,10 +3484,10 @@ // (this is not a condition that holds when we set the region up, // as the region is not supposed to be empty in the first place) _finger = bottom; - } else if (limit >= _region_limit) { - assert(limit >= _finger, "peace of mind"); + } else if (tams >= _region_limit) { + assert(tams >= _finger, "peace of mind"); + // leave _finger unchanged } else { - assert(limit < _region_limit, "only way to get here"); // This can happen under some pretty unusual circumstances. An // evacuation pause empties the region underneath our feet (NTAMS // at bottom). We then do some allocation in the region (NTAMS @@ -3466,10 +3498,10 @@ // and we do not need in fact to scan anything else. So, we simply // set _finger to be limit to ensure that the bitmap iteration // doesn't do anything. - _finger = limit; + _finger = tams; } - _region_limit = limit; + _region_limit = tams; } void CMTask::giveup_current_region() { @@ -3770,6 +3802,7 @@ (void*) obj); } + assert(obj != NULL, "invariant"); assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); assert(!_g1h->is_on_master_free_list( _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); @@ -4058,7 +4091,6 @@ assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); assert(concurrent() == _cm->concurrent(), "they should be the same"); - G1CollectorPolicy* g1_policy = _g1h->g1_policy(); assert(_task_queues != NULL, "invariant"); assert(_task_queue != NULL, "invariant"); assert(_task_queues->queue(_worker_id) == _task_queue, "invariant"); @@ -4081,8 +4113,7 @@ // and do_marking_step() is not being called serially. bool do_stealing = do_termination && !is_serial; - double diff_prediction_ms = - g1_policy->get_new_prediction(&_marking_step_diffs_ms); + double diff_prediction_ms = _g1h->g1_policy()->get_new_prediction(&_marking_step_diffs_ms); _time_target_ms = time_target_ms - diff_prediction_ms; // set up the variables that are used in the work-based scheme to @@ -4131,8 +4162,7 @@ do { if (!has_aborted() && _curr_region != NULL) { // This means that we're already holding on to a region. - assert(_finger != NULL, "if region is not NULL, then the finger " - "should not be NULL either"); + assert(_finger != NULL, "if region is not NULL, then the finger should not be NULL either"); // We might have restarted this task after an evacuation pause // which might have evacuated the region we're holding on to @@ -4156,8 +4186,8 @@ HR_FORMAT_PARAMS(_curr_region)); } - assert(!_curr_region->isHumongous() || mr.start() == _curr_region->bottom(), - "humongous regions should go around loop once only"); + //assert(!_curr_region->isHumongous() || mr.start() == _curr_region->bottom(), + // "humongous regions should go around loop once only"); // Some special cases: // If the memory region is empty, we can just give up the region. diff --git a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp +++ b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp @@ -88,7 +88,7 @@ size_t region_size_bytes = mr.byte_size(); uint index = hr->hrs_index(); - assert(!hr->continuesHumongous(), "should not be HC region"); + //assert(!hr->continuesHumongous(), "should not be HC region"); assert(hr == g1h->heap_region_containing(start), "sanity"); assert(hr == g1h->heap_region_containing(mr.last()), "sanity"); assert(marked_bytes_array != NULL, "pre-condition"); @@ -203,6 +203,7 @@ // object and so have to supply it. inline bool ConcurrentMark::par_mark_and_count(oop obj, uint worker_id) { HeapWord* addr = (HeapWord*)obj; + assert(addr != NULL, "invariant"); HeapRegion* hr = _g1h->heap_region_containing_raw(addr); return par_mark_and_count(obj, hr, worker_id); } @@ -270,6 +271,7 @@ inline void CMTask::push(oop obj) { HeapWord* objAddr = (HeapWord*) obj; + assert(obj != NULL, "invariant"); assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); assert(!_g1h->is_on_master_free_list( _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); @@ -410,9 +412,7 @@ assert(hr->is_in(addr), "pre-condition"); } assert(hr != NULL, "sanity"); - // Given that we're looking for a region that contains an object - // header it's impossible to get back a HC region. - assert(!hr->continuesHumongous(), "sanity"); + assert(hr->has_own_objects(), "sanity"); // We cannot assert that word_size == obj->size() given that obj // might not be in a consistent state (another thread might be in diff --git a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp @@ -659,6 +659,16 @@ return _next_offset_threshold; } +HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold(HeapWord* bottom) { + assert(!Universe::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + _next_offset_index = _array->index_for(bottom); + _next_offset_index++; + _next_offset_threshold = + _array->address_for_index(_next_offset_index); + return _next_offset_threshold; +} + void G1BlockOffsetArrayContigSpace::zero_bottom_entry() { assert(!Universe::heap()->is_in_reserved(_array->_offset_array), "just checking"); diff --git a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp @@ -501,6 +501,7 @@ // Initialize the threshold to reflect the first boundary after the // bottom of the covered region. HeapWord* initialize_threshold(); + HeapWord* initialize_threshold(HeapWord* bottom); // Zero out the entry for _bottom (offset will be zero). void zero_bottom_entry(); diff --git a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @@ -56,6 +56,7 @@ #include "memory/referenceProcessor.hpp" #include "oops/oop.inline.hpp" #include "oops/oop.pcgc.inline.hpp" +#include "runtime/timer.hpp" #include "runtime/vmThread.hpp" #include "utilities/ticks.hpp" @@ -446,7 +447,7 @@ // all nmethods must be scanned during a partial collection. bool G1CollectedHeap::is_in_partial_collection(const void* p) { HeapRegion* hr = heap_region_containing(p); - return hr != NULL && hr->in_collection_set(); + return hr->in_collection_set(); } #endif @@ -456,13 +457,7 @@ G1CollectedHeap* g1h = G1CollectedHeap::heap(); G1CollectorPolicy* g1p = g1h->g1_policy(); HeapRegion* hr = heap_region_containing(p); - if (hr == NULL) { - // null - assert(p == NULL, err_msg("Not NULL " PTR_FORMAT ,p)); - return false; - } else { - return !hr->isHumongous(); - } + return !hr->startsHumongous() && hr->has_own_objects(); } void G1CollectedHeap::check_ct_logs_at_safepoint() { @@ -555,7 +550,7 @@ return NULL; } -HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand) { +HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand, bool use_partially_free_regions) { assert(!isHumongous(word_size) || word_size <= HeapRegion::GrainWords, "the only time we use this to allocate a humongous region is " "when we are allocating a single humongous region"); @@ -574,6 +569,35 @@ } } + // First check the partially free regions. + if (use_partially_free_regions) { + res = _partially_free_regions_set.head(); + if (res != NULL) { + assert(res->isHumongous(), err_msg("Region %u should be humongous since it is on the partially free list", res->hrs_index())); + if (word_size < (res->free() / HeapWordSize)) { + _partially_free_regions_set.remove_head(); + // We need to make sure that objects allocated after the humongous object do not have overlapping cards with + // that object. Simplest solution is to align up the top() to a card boundary. + // TODO: do we need to add a filler object between old top and new top? + HeapWord* old_top = res->top(); + HeapWord* card_aligned_top = (HeapWord*)(align_pointer_up(old_top, CardTableModRefBS::card_size)); + size_t alignment_waste = pointer_delta(card_aligned_top, old_top, 1); + res->set_top(card_aligned_top); + res->set_first_object_start(card_aligned_top); + res->initialize_threshold(); + res->cross_threshold(res->bottom(), res->top()); + _summary_bytes_used += alignment_waste; + if (G1VerboseAllocObjsInHCRegions) { + gclog_or_tty->print_cr("Using partially free region %u (%u), alignment waste: " SIZE_FORMAT, + res->hrs_index(), res->humongous_start_region()->hrs_index(), alignment_waste); + } + return res; + } else { + res = NULL; + } + } + } + res = _free_list.remove_region(is_old); if (res == NULL) { @@ -618,7 +642,7 @@ // Only one region to allocate, no need to go through the slower // path. The caller will attempt the expansion if this fails, so // let's not try to expand here too. - HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); + HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */, false /* partially free regions */); if (hr != NULL) { first = hr->hrs_index(); } else { @@ -649,6 +673,9 @@ } } } + if (first != G1_NULL_HRS_INDEX) { + region_at(first)->set_humongous_allocated_at(total_collections(), total_full_collections()); + } return first; } @@ -715,6 +742,7 @@ hr = region_at(i); hr->set_continuesHumongous(first_hr); } + // If we have "continues humongous" regions (hr != NULL), then the // end of the last one should match new_end. assert(hr == NULL || hr->end() == new_end, "sanity"); @@ -764,6 +792,13 @@ assert(hr->bottom() < new_top && new_top <= hr->end(), "new_top should fall on this region"); hr->set_top(new_top); + if (G1AllocObjsInHCRegions) { + // TODO: We should probably only add regions that have more than certain amount of free memory + _partially_free_regions_set.add_ordered(hr); + if (G1VerboseAllocObjsInHCRegions) { + gclog_or_tty->print_cr("Adding partially free region %u (%u)", hr->hrs_index(), hr->humongous_start_region()->hrs_index()); + } + } _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top); } else { // not last one @@ -1172,13 +1207,15 @@ bool doHeapRegion(HeapRegion* r) { HeapRegionRemSet* hrrs = r->rem_set(); - if (r->continuesHumongous()) { + /* + * TODO: last humongous regions that were just cleared may have a hrss. + if (r->continuesHumongous() && !r->has_own_objects()) { // We'll assert that the strong code root list and RSet is empty assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); assert(hrrs->occupied() == 0, "RSet should be empty"); return false; } - +*/ _g1h->reset_gc_time_stamps(r); hrrs->clear(); // You might think here that we could clear just the cards @@ -1209,7 +1246,7 @@ { } bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { + if (r->has_own_objects()) { _cl.set_from(r); r->oop_iterate(&_cl); } @@ -1786,7 +1823,7 @@ HeapWord* new_end = (HeapWord*) _g1_storage.high(); update_committed_space(old_end, new_end); - FreeRegionList expansion_list("Local Expansion List"); + FreeRegionList expansion_list("Local Expansion List", false, true); MemRegion mr = _hrs.expand_by(old_end, new_end, &expansion_list); assert(mr.start() == old_end, "post-condition"); // mr might be a smaller region than what was requested if @@ -1919,10 +1956,12 @@ _g1mm(NULL), _refine_cte_cl(NULL), _full_collection(false), - _free_list("Master Free List", new MasterFreeRegionListMtSafeChecker()), - _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), + _free_list("Master Free List", false, true, new MasterFreeRegionListMtSafeChecker()), + _secondary_free_list("Secondary Free List", false, true, new SecondaryFreeRegionListMtSafeChecker()), _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), + _humongous_is_live(), + _partially_free_regions_set("Partially Free Regions", true, false, NULL), _free_regions_coming(false), _young_list(new YoungList(this)), _gc_time_stamp(0), @@ -1934,8 +1973,7 @@ _old_marking_cycles_started(0), _old_marking_cycles_completed(0), _concurrent_cycle_started(false), - _in_cset_fast_test(NULL), - _in_cset_fast_test_base(NULL), + _in_cset_fast_test(), _dirty_cards_region_list(NULL), _worker_cset_start_region(NULL), _worker_cset_start_region_time_stamp(NULL), @@ -1973,6 +2011,10 @@ NOT_PRODUCT(reset_evacuation_should_fail();) guarantee(_task_queues != NULL, "task_queues allocation failure."); + + Copy::zero_to_bytes(_humongous_freed_any_gc_stats, sizeof(size_t) * (HUMONGOUS_FREED_NUM_ENTRIES + 1)); + Copy::zero_to_bytes(_humongous_freed_full_gc_stats, sizeof(size_t) * (HUMONGOUS_FREED_NUM_ENTRIES + 1)); + Copy::zero_to_bytes(_humongous_object_type, sizeof(size_t) * 2); } jint G1CollectedHeap::initialize() { @@ -2077,20 +2119,8 @@ _g1h = this; - _in_cset_fast_test_length = max_regions(); - _in_cset_fast_test_base = - NEW_C_HEAP_ARRAY(bool, (size_t) _in_cset_fast_test_length, mtGC); - - // We're biasing _in_cset_fast_test to avoid subtracting the - // beginning of the heap every time we want to index; basically - // it's the same with what we do with the card table. - _in_cset_fast_test = _in_cset_fast_test_base - - ((uintx) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes); - - // Clear the _cset_fast_test bitmap in anticipation of adding - // regions to the incremental collection set for the first - // evacuation pause. - clear_cset_fast_test(); + _in_cset_fast_test.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); + _humongous_is_live.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); // Create the ConcurrentMark data structure and thread. // (Must do this late, so that "max_regions" is defined.) @@ -2178,6 +2208,10 @@ return JNI_OK; } +void G1CollectedHeap::clear_humongous_is_live_table() { + _humongous_is_live.clear(); +} + size_t G1CollectedHeap::conservative_max_heap_alignment() { return HeapRegion::max_region_size(); } @@ -2267,7 +2301,11 @@ } void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) { - assert(!hr->continuesHumongous(), "pre-condition"); + //assert(!hr->continuesHumongous(), "pre-condition"); + /*if (hr->continuesHumongous()) { + assert(hr->has_own_objects(), "only reason to get here with a HC region"); + return; + }*/ hr->reset_gc_time_stamp(); if (hr->startsHumongous()) { uint first_index = hr->hrs_index() + 1; @@ -2353,9 +2391,7 @@ public: SumUsedClosure() : _used(0) {} bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { - _used += r->used(); - } + _used += r->used(); return false; } size_t result() { return _used; } @@ -2591,7 +2627,7 @@ IterateOopClosureRegionClosure(MemRegion mr, ExtendedOopClosure* cl) : _mr(mr), _cl(cl) {} bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { + if (r->has_own_objects()) { r->oop_iterate(_cl); } return false; @@ -2615,7 +2651,7 @@ public: IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} bool doHeapRegion(HeapRegion* r) { - if (! r->continuesHumongous()) { + if (r->has_own_objects()) { r->object_iterate(_cl); } return false; @@ -2695,7 +2731,7 @@ // if the region has already been claimed or it's not // "continues humongous" we're done if (chr->claim_value() == claim_value || - !chr->continuesHumongous()) { + !chr->continuesHumongous()) { //TODO: Why could it possibly be !continues? break; } @@ -2704,22 +2740,17 @@ assert(chr->claim_value() != claim_value, "sanity"); assert(chr->humongous_start_region() == r, "sanity"); - if (chr->claimHeapRegion(claim_value)) { - // we should always be able to claim it; no one else should - // be trying to claim this region - - bool res2 = cl->doHeapRegion(chr); - assert(!res2, "Should not abort"); - - // Right now, this holds (i.e., no closure that actually - // does something with "continues humongous" regions - // clears them). We might have to weaken it in the future, - // but let's leave these two asserts here for extra safety. - assert(chr->continuesHumongous(), "should still be the case"); - assert(chr->humongous_start_region() == r, "sanity"); - } else { - guarantee(false, "we should not reach here"); - } + guarantee(chr->claimHeapRegion(claim_value), "we should always be able to claim it; no one else should be trying to claim this region"); + + bool res2 = cl->doHeapRegion(chr); + assert(!res2, "Should not abort"); + + // Right now, this holds (i.e., no closure that actually + // does something with "continues humongous" regions + // clears them). We might have to weaken it in the future, + // but let's leave these two asserts here for extra safety. + assert(chr->continuesHumongous(), "should still be the case"); + assert(chr->humongous_start_region() == r, "sanity"); } } @@ -2963,21 +2994,17 @@ Space* G1CollectedHeap::space_containing(const void* addr) const { - Space* res = heap_region_containing(addr); - return res; + assert(addr != NULL, "invariant"); + return heap_region_containing(addr); } HeapWord* G1CollectedHeap::block_start(const void* addr) const { Space* sp = space_containing(addr); - if (sp != NULL) { - return sp->block_start(addr); - } - return NULL; + return sp->block_start(addr); } size_t G1CollectedHeap::block_size(const HeapWord* addr) const { Space* sp = space_containing(addr); - assert(sp != NULL, "block_size of address outside of heap"); return sp->block_size(addr); } @@ -3321,21 +3348,28 @@ } bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { + assert(r != NULL, "precondition"); + if (r->has_own_objects()) { bool failures = false; r->verify(_vo, &failures); if (failures) { _failures = true; } else { VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); - r->object_iterate(¬_dead_yet_cl); + size_t max_live_bytes = r->max_live_bytes(); + if (r->continuesHumongous()) { + r->non_humongous_space()->object_iterate(¬_dead_yet_cl); + max_live_bytes += pointer_delta(r->humongous_start_region()->top(), r->bottom(), 1); + } else { + r->object_iterate(¬_dead_yet_cl); + } if (_vo != VerifyOption_G1UseNextMarking) { - if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { + if (max_live_bytes < not_dead_yet_cl.live_bytes()) { gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] " "max_live_bytes "SIZE_FORMAT" " "< calculated "SIZE_FORMAT, r->bottom(), r->end(), - r->max_live_bytes(), + max_live_bytes, not_dead_yet_cl.live_bytes()); _failures = true; } @@ -3680,6 +3714,10 @@ (total_collections() % G1SummarizeRSetStatsPeriod == 0)) { g1_rem_set()->print_periodic_summary_info("Before GC RS summary"); } + + if (G1ReclaimDeadLargeObjectsAtYoungGC) { + clear_humongous_is_live_table(); + } } void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) { @@ -3690,6 +3728,9 @@ ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) { g1_rem_set()->print_periodic_summary_info("After GC RS summary"); } + if (G1TraceHumongousObjectLiveness) { + print_freed_humongous_region_stats(); + } // FIXME: what is this about? // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled" @@ -4045,6 +4086,12 @@ g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); + for (uint i = 0; i < n_regions(); i++) { + if (region_at(i)->startsHumongous()) { + _in_cset_fast_test.set_humongous(i); + } + } + _cm->note_start_of_gc(); // We should not verify the per-thread SATB buffers given that // we have not filtered them yet (we'll do so during the @@ -4095,6 +4142,9 @@ true /* verify_fingers */); free_collection_set(g1_policy()->collection_set(), evacuation_info); + if (G1ReclaimDeadLargeObjectsAtYoungGC) { + eagerly_reclaim_humongous_regions(); + } g1_policy()->clear_collection_set(); cleanup_surviving_young_words(); @@ -4440,6 +4490,7 @@ while (_evac_failure_scan_stack->length() > 0) { oop obj = _evac_failure_scan_stack->pop(); + assert(obj != NULL, "invariant"); _evac_failure_closure->set_region(heap_region_containing(obj)); obj->oop_iterate_backwards(_evac_failure_closure); } @@ -4493,6 +4544,7 @@ void G1CollectedHeap::handle_evacuation_failure_common(oop old, markOop m) { preserve_mark_if_necessary(old, m); + assert(old != NULL, "invariant"); HeapRegion* r = heap_region_containing(old); if (!r->evacuation_failed()) { r->set_evacuation_failed(true); @@ -4672,8 +4724,8 @@ void G1ParCopyHelper::mark_object(oop obj) { #ifdef ASSERT + assert(obj != NULL, "invariant"); HeapRegion* hr = _g1->heap_region_containing(obj); - assert(hr != NULL, "sanity"); assert(!hr->in_collection_set(), "should not mark objects in the CSet"); #endif // ASSERT @@ -4688,11 +4740,9 @@ assert(from_obj != to_obj, "should not be self-forwarded"); HeapRegion* from_hr = _g1->heap_region_containing(from_obj); - assert(from_hr != NULL, "sanity"); assert(from_hr->in_collection_set(), "from obj should be in the CSet"); HeapRegion* to_hr = _g1->heap_region_containing(to_obj); - assert(to_hr != NULL, "sanity"); assert(!to_hr->in_collection_set(), "should not mark objects in the CSet"); #endif // ASSERT @@ -4728,6 +4778,12 @@ #endif // !PRODUCT if (obj_ptr == NULL) { + // The allocation failure may have been caused by attempted allocation of a + // humongous object. Detect this and process appropriately. + if (_g1h->isHumongous(word_sz)) { + _g1h->set_humongous_is_live((HeapWord*)old); + return NULL; + } // This will either forward-to-self, or detect that someone else has // installed a forwarding pointer. return _g1h->handle_evacuation_failure_par(this, old); @@ -4824,25 +4880,31 @@ assert(_worker_id == _par_scan_state->queue_num(), "sanity"); - if (_g1->in_cset_fast_test(obj)) { + bool needs_marking = true; + + if (_g1->is_in_cset_or_humongous(obj)) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); } else { forwardee = _par_scan_state->copy_to_survivor_space(obj); } - assert(forwardee != NULL, "forwardee should not be NULL"); - oopDesc::encode_store_heap_oop(p, forwardee); - if (do_mark_object && forwardee != obj) { - // If the object is self-forwarded we don't need to explicitly - // mark it, the evacuation failure protocol will do so. - mark_forwarded_object(obj, forwardee); + if (forwardee != NULL) { + oopDesc::encode_store_heap_oop(p, forwardee); + if (do_mark_object && forwardee != obj) { + // If the object is self-forwarded we don't need to explicitly + // mark it, the evacuation failure protocol will do so. + mark_forwarded_object(obj, forwardee); + } + + if (barrier == G1BarrierKlass) { + do_klass_barrier(p, forwardee); + } + needs_marking = false; } - - if (barrier == G1BarrierKlass) { - do_klass_barrier(p, forwardee); - } - } else { + } + + if (needs_marking) { // The object is not in collection set. If we're a root scanning // closure during an initial mark pause (i.e. do_mark_object will // be true) then attempt to mark the object. @@ -4948,7 +5010,7 @@ RefToScanQueueSet *_queues; ParallelTaskTerminator _terminator; uint _n_workers; - + Mutex _stats_lock; Mutex* stats_lock() { return &_stats_lock; } @@ -4965,7 +5027,7 @@ _queues(task_queues), _terminator(0, _queues), _stats_lock(Mutex::leaf, "parallel G1 stats lock", true) - {} + { } RefToScanQueueSet* queues() { return _queues; } @@ -5031,13 +5093,17 @@ // region's RSet. int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings; + G1GCPhaseTimes* pt = _g1h->g1_policy()->phase_times(); + GCPhaseTimeTracker phase_tracker(pt->get_ext_root_scan_phase_times(), pt->num_ext_root_scan_phases(), worker_id); + pss.start_strong_roots(); _g1h->g1_process_strong_roots(/* is scavenging */ true, SharedHeap::ScanningOption(so), scan_root_cl, &push_heap_rs_cl, scan_klasses_cl, - worker_id); + worker_id, + &phase_tracker); pss.end_strong_roots(); { @@ -5080,7 +5146,8 @@ OopClosure* scan_non_heap_roots, OopsInHeapRegionClosure* scan_rs, G1KlassScanClosure* scan_klasses, - int worker_i) { + int worker_i, + GCPhaseTimeTracker* phase_tracker) { // First scan the strong roots double ext_roots_start = os::elapsedTime(); @@ -5091,16 +5158,19 @@ process_strong_roots(false, // no scoping; this is parallel code so, &buf_scan_non_heap_roots, - scan_klasses - ); - - // Now the CM ref_processor roots. - if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) { - // We need to treat the discovered reference lists of the - // concurrent mark ref processor as roots and keep entries - // (which are added by the marking threads) on them live - // until they can be processed at the end of marking. - ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots); + scan_klasses, + phase_tracker); + + { + // Now the CM ref_processor roots. + TrackPhaseTime x(phase_tracker, SharedHeap::SH_PS_NumElements + G1H_PS_refProcessor_oops_do); + if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) { + // We need to treat the discovered reference lists of the + // concurrent mark ref processor as roots and keep entries + // (which are added by the marking threads) on them live + // until they can be processed at the end of marking. + ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots); + } } // Finish up any enqueued closure apps (attributed as object copy time). @@ -5119,13 +5189,17 @@ // to make sure we remove any oops into the CSet (which will show up // as implicitly live). double satb_filtering_ms = 0.0; - if (!_process_strong_tasks->is_task_claimed(G1H_PS_filter_satb_buffers)) { - if (mark_in_progress()) { - double satb_filter_start = os::elapsedTime(); - - JavaThread::satb_mark_queue_set().filter_thread_buffers(); - - satb_filtering_ms = (os::elapsedTime() - satb_filter_start) * 1000.0; + { + TrackPhaseTime x(phase_tracker, SharedHeap::SH_PS_NumElements + G1H_PS_filter_satb_buffers); + if (!_process_strong_tasks->is_task_claimed(G1H_PS_filter_satb_buffers)) { + + if (mark_in_progress()) { + double satb_filter_start = os::elapsedTime(); + + JavaThread::satb_mark_queue_set().filter_thread_buffers(); + + satb_filtering_ms = (os::elapsedTime() - satb_filter_start) * 1000.0; + } } } g1_policy()->phase_times()->record_satb_filtering_time(worker_i, satb_filtering_ms); @@ -5303,10 +5377,10 @@ public: G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} void do_oop(narrowOop* p) { guarantee(false, "Not needed"); } - void do_oop( oop* p) { + void do_oop(oop* p) { oop obj = *p; - if (_g1->obj_in_cs(obj)) { + if (_g1->is_in_cset(obj)) { assert( obj->is_forwarded(), "invariant" ); *p = obj->forwardee(); } @@ -5341,7 +5415,7 @@ template void do_oop_work(T* p) { oop obj = oopDesc::load_decode_heap_oop(p); - if (_g1h->obj_in_cs(obj)) { + if (_g1h->is_in_cset_or_humongous(obj)) { // If the referent object has been forwarded (either copied // to a new location or to itself in the event of an // evacuation failure) then we need to update the reference @@ -5842,7 +5916,7 @@ g1_par_task.work(0); } end_par_time_sec = os::elapsedTime(); - + // Closing the inner scope will execute the destructor // for the StrongRootsScope object. We record the current // elapsed time before closing the scope so that time @@ -5942,7 +6016,39 @@ free_list->add_ordered(hr); } -void G1CollectedHeap::free_humongous_region(HeapRegion* hr, +void G1CollectedHeap::update_freed_humongous_region_stats(HeapRegion* hr) { + uint duration_gc = total_collections() - hr->humongous_allocated_at_gc(); + if (duration_gc < HUMONGOUS_FREED_NUM_ENTRIES) { + _humongous_freed_any_gc_stats[duration_gc]++; + } else { + _humongous_freed_any_gc_stats[HUMONGOUS_FREED_NUM_ENTRIES]++; + } + + uint duration_full_gc = total_full_collections() - hr->humongous_allocated_at_full_gc(); + if (duration_full_gc < HUMONGOUS_FREED_NUM_ENTRIES) { + _humongous_freed_full_gc_stats[duration_full_gc]++; + } else { + _humongous_freed_full_gc_stats[HUMONGOUS_FREED_NUM_ENTRIES]++; + } + + _humongous_object_type[ ((oop)hr->bottom())->is_objArray() ? 1 : 0 ]++; +} + +void G1CollectedHeap::print_freed_humongous_region_stats(outputStream* out) { + out->print_cr("Humongous objects allocated (prim: %d ref: %d)", _humongous_object_type[0], _humongous_object_type[1]); + out->print("full gc"); + for (int i = 0; i < (HUMONGOUS_FREED_NUM_ENTRIES+1); i++) { + out->print(" "SIZE_FORMAT, _humongous_freed_full_gc_stats[i]); + } + out->cr(); + out->print("any gc"); + for (int i = 0; i < (HUMONGOUS_FREED_NUM_ENTRIES+1); i++) { + out->print(" "SIZE_FORMAT, _humongous_freed_any_gc_stats[i]); + } + out->cr(); +} + +HeapRegion* G1CollectedHeap::free_humongous_region(HeapRegion* hr, FreeRegionList* free_list, bool par) { assert(hr->startsHumongous(), "this is only for starts humongous regions"); @@ -5952,6 +6058,10 @@ // We need to read this before we make the region non-humongous, // otherwise the information will be gone. uint last_index = hr->last_hc_index(); + + HeapRegion* last_region = region_at(last_index - 1); + bool last_region_has_own_objects = last_region->first_object_start() > last_region->bottom(); + hr->set_notHumongous(); free_region(hr, free_list, par); @@ -5960,9 +6070,27 @@ HeapRegion* curr_hr = region_at(i); assert(curr_hr->continuesHumongous(), "invariant"); curr_hr->set_notHumongous(); - free_region(curr_hr, free_list, par); + if (curr_hr == last_region) { + if (last_region_has_own_objects) { + assert(curr_hr->containing_set() != &_partially_free_regions_set, err_msg("Region %u must have been popped from the partially free regions", curr_hr->hrs_index())); + //This is the last continues region and it has other objects allocated into it. + // It would have been nice to have this assert here: + // assert(curr_hr->containing_set() == &_old_set, "Should have been added to old_set when retired"); + // but this code is executed between tear_down_region sets and rebulid_region sets for the full GC. + // In that case the containing set is empty. + } else { + { + MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); + _partially_free_regions_set.remove(curr_hr); + } + free_region(curr_hr, free_list, par); + } + } else { + free_region(curr_hr, free_list, par); + } i += 1; } + return last_region_has_own_objects ? last_region : NULL; } void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, @@ -6072,7 +6200,7 @@ G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); double start = os::elapsedTime(); - { + if (_g1h->_dirty_cards_region_list != NULL) { // Iterate over the dirty cards region list. G1ParCleanupCTTask cleanup_task(ct_bs, this); @@ -6106,7 +6234,7 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info) { size_t pre_used = 0; - FreeRegionList local_free_list("Local List for CSet Freeing"); + FreeRegionList local_free_list("Local List for CSet Freeing", false, true); double young_time_ms = 0.0; double non_young_time_ms = 0.0; @@ -6215,6 +6343,119 @@ policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); } +void G1CollectedHeap::check_partial() { +#ifdef ASSERT + HeapRegion* hr = _partially_free_regions_set.head(); + while (hr != NULL) { + guarantee(hr->isHumongous(), err_msg("Partial check failed for %u", hr->hrs_index())); + hr = hr->next(); + } +#endif +} + +class G1FreeHumongousRegionClosure : public HeapRegionClosure { + private: + FreeRegionList* _free_region_list; + HeapRegionSet* _proxy_set; + HeapRegionSetCount _humongous_regions_removed; + size_t _freed_bytes; + public: + + G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) : + _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + if (!r->startsHumongous()) { + return false; + } + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + if (g1h->humongous_is_live(r->hrs_index()) || + !r->rem_set()->is_empty() || + oop(r->bottom())->is_objArray() + ) { + + if (G1TraceReclaimDeadLargeObjectsAtYoungGC) { + gclog_or_tty->print_cr("Live humongous %d region %d with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", + r->isHumongous(), + r->hrs_index(), r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), + g1h->humongous_is_live(r->hrs_index()), + oop(r->bottom())->is_objArray() + ); + } + + return false; + } + + if (G1TraceReclaimDeadLargeObjectsAtYoungGC) { + gclog_or_tty->print_cr("Reclaim humongous %d start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", + r->isHumongous(), r->bottom(), + r->hrs_index(), r->region_num(), + r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), + g1h->humongous_is_live(r->hrs_index()), + oop(r->bottom())->is_objArray()); + } + + HeapWord* hum_top = r->top(); + _freed_bytes += r->used(); + r->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, r->capacity()); + HeapRegion* last_region = G1CollectedHeap::heap()->free_humongous_region(r, _free_region_list, false); + + if (last_region != NULL) { + last_region->make_old(); + // We didn't completely free all of the humongous object up. Add back the piece of the humongous + // object that we still use. + _freed_bytes -= byte_size(last_region->bottom(), hum_top); + } + + g1h->check_partial(); + + return false; + } + + HeapRegionSetCount& humongous_free_count() { + return _humongous_regions_removed; + } + + size_t bytes_freed() const { + return _freed_bytes; + } +}; + +void G1CollectedHeap::eagerly_reclaim_humongous_regions() { + guarantee(G1ReclaimDeadLargeObjectsAtYoungGC, "Feature must be enabled"); + double start_time = os::elapsedTime(); + + FreeRegionList local_cleanup_list("Local Humongous Cleanup List", false, true, new HumongousRegionSetMtSafeChecker()); + + G1FreeHumongousRegionClosure cl(&local_cleanup_list); + heap_region_iterate(&cl); + + HeapRegionSetCount empty_set; + remove_from_old_sets(empty_set, cl.humongous_free_count()); + + prepend_to_freelist(&local_cleanup_list); + decrement_summary_bytes(cl.bytes_freed()); + + G1HRPrinter* hr_printer = _g1h->hr_printer(); + if (hr_printer->is_active()) { + FreeRegionListIterator iter(&local_cleanup_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hr_printer->cleanup(hr); + } + } + + *g1_policy()->phase_times()->fast_reclaim_humongous_time_ms_addr() = (os::elapsedTime() - start_time) * 1000.0; +} + // This routine is similar to the above but does not record // any policy statistics or update free lists; we are abandoning // the current incremental collection set in preparation of a @@ -6334,7 +6575,11 @@ // We ignore young regions, we'll empty the young list afterwards } else if (r->isHumongous()) { // We ignore humongous regions, we're not tearing down the - // humongous region set + // humongous region set but we need to remove any regions that + // may have been added to the old_set + if (r->continuesHumongous() && r->has_own_objects()) { + _old_set->remove(r); + } } else { // The rest should be old _old_set->remove(r); @@ -6343,7 +6588,7 @@ } ~TearDownRegionSetsClosure() { - assert(_old_set->is_empty(), "post-condition"); + assert(_old_set->is_empty(), err_msg("OldSet should be empty but has %u entries", _old_set->length())); } }; @@ -6358,6 +6603,8 @@ // the first step when rebuilding the regions sets again. The reason for // this is that during a full GC string deduplication needs to know if // a collected region was young or old when the full GC was initiated. + + _partially_free_regions_set.remove_all(); } _free_list.remove_all(); } @@ -6381,21 +6628,20 @@ } bool doHeapRegion(HeapRegion* r) { - if (r->continuesHumongous()) { - return false; - } - if (r->is_empty()) { - // Add free regions to the free list - _free_list->add_as_tail(r); + if (!r->isHumongous()) { + // Add free regions to the free list + _free_list->add_as_tail(r); + } } else if (!_free_list_only) { assert(!r->is_young(), "we should not come across young regions"); - if (r->isHumongous()) { - // We ignore humongous regions, we left the humongous set unchanged - } else { + if (r->startsHumongous()) { + // We ignore starts humongous regions, we left the humongous set unchanged + } else if (r->has_own_objects()) { // The rest should be old, add them to the old set _old_set->add(r); + // TODO: We should also add the last continues region to the _partially_free_region_set } _total_used += r->used(); } @@ -6432,12 +6678,9 @@ } bool G1CollectedHeap::is_in_closed_subset(const void* p) const { + assert(p != NULL, "invariant"); HeapRegion* hr = heap_region_containing(p); - if (hr == NULL) { - return false; - } else { - return hr->is_in(p); - } + return hr->is_in(p); } // Methods for the mutator alloc region @@ -6451,7 +6694,8 @@ if (force || !young_list_full) { HeapRegion* new_alloc_region = new_region(word_size, false /* is_old */, - false /* do_expand */); + false /* do_expand */, + false /* use_partially_free_regions */); if (new_alloc_region != NULL) { set_region_short_lived_locked(new_alloc_region); _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full); @@ -6510,9 +6754,11 @@ if (count < g1_policy()->max_regions(ap)) { bool survivor = (ap == GCAllocForSurvived); + bool use_partially_free_regions = G1AllocObjsInHCRegions && ap == GCAllocForTenured; HeapRegion* new_alloc_region = new_region(word_size, !survivor, - true /* do_expand */); + true /* do_expand */, + use_partially_free_regions); if (new_alloc_region != NULL) { // We really only need to do this for old regions given that we // should never scan survivors. But it doesn't hurt to do it @@ -6591,12 +6837,9 @@ _old_count(), _humongous_count(), _free_count(){ } bool doHeapRegion(HeapRegion* hr) { - if (hr->continuesHumongous()) { - return false; - } - - if (hr->is_young()) { - // TODO + if (!hr->has_own_objects() || hr->is_young()) { + // We currently don't track continues humongous regions (unless we have allocated into them) + // or young regions in any sets } else if (hr->startsHumongous()) { assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->region_num())); _humongous_count.increment(1u, hr->capacity()); @@ -6685,7 +6928,8 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); HeapRegion* hr = _g1h->heap_region_containing(obj); - assert(!hr->continuesHumongous(), + //assert(!hr->continuesHumongous(), // TODO: need to allow C-hum regions with objects here? + assert(hr->has_own_objects(), err_msg("trying to add code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT " starting at "HR_FORMAT, _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); @@ -6717,7 +6961,8 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); HeapRegion* hr = _g1h->heap_region_containing(obj); - assert(!hr->continuesHumongous(), + //assert(!hr->continuesHumongous(), // TODO: need to allow C-hum regions with objects here? + assert(hr->has_own_objects(), err_msg("trying to remove code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT " starting at "HR_FORMAT, _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); @@ -6845,7 +7090,8 @@ bool doHeapRegion(HeapRegion *hr) { HeapRegionRemSet* hrrs = hr->rem_set(); - if (hr->continuesHumongous()) { + //if (hr->continuesHumongous()) { // TODO: need to allow C-hum regions with objects here? + if (!hr->has_own_objects()) { // Code roots should never be attached to a continuation of a humongous region assert(hrrs->strong_code_roots_list_length() == 0, err_msg("code roots should never be attached to continuations of humongous region "HR_FORMAT diff --git a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp @@ -41,6 +41,7 @@ #include "memory/memRegion.hpp" #include "memory/sharedHeap.hpp" #include "utilities/stack.hpp" +#include "g1GCPhaseTimes.hpp" // A "G1CollectedHeap" is an implementation of a java heap for HotSpot. // It uses the "Garbage First" heap organization and algorithm, which @@ -197,6 +198,27 @@ bool do_object_b(oop p); }; +// Instances of this class are used for quick tests on whether a reference points +// into the collection set. Each of the array's elements denotes whether the +// corresponding region is in the collection set. +class G1FastCSetBiasedMappedArray : public G1BiasedMappedArray { + private: + enum { + InNeither, // neither in collection set nor humongous + InCSet, // region is in collection set only + IsHumongous // region is a humongous start region + }; + protected: + char default_value() const { return InNeither; } + public: + void set_humongous(uintptr_t index) { assert(get_by_index(index) != InCSet, "Should not overwrite InCSet values"); set_by_index(index, IsHumongous); } + void set_in_cset(uintptr_t index) { assert(get_by_index(index) != IsHumongous, "Should not overwrite InCSetOrHumongous value"); set_by_index(index, InCSet); } + + bool is_in_cset_or_humongous(HeapWord* addr) const { return get_by_address(addr) != InNeither; } + bool is_in_cset(HeapWord* addr) const { return get_by_address(addr) == InCSet; } + void clear() { G1BiasedMappedArray::clear(); } +}; + class RefineCardTableEntryClosure; class G1CollectedHeap : public SharedHeap { @@ -226,6 +248,7 @@ friend class EvacPopObjClosure; friend class G1ParCleanupCTTask; + friend class G1FreeHumongousRegionClosure; // Other related classes. friend class G1MarkSweep; @@ -256,6 +279,11 @@ // It keeps track of the humongous regions. HeapRegionSet _humongous_set; + FreeRegionList _partially_free_regions_set; + + void clear_humongous_is_live_table(); + void eagerly_reclaim_humongous_regions(); + // The number of regions we could create by expansion. uint _expansion_regions; @@ -353,26 +381,19 @@ // than the current allocation region. size_t _summary_bytes_used; - // This is used for a quick test on whether a reference points into - // the collection set or not. Basically, we have an array, with one - // byte per region, and that byte denotes whether the corresponding - // region is in the collection set or not. The entry corresponding - // the bottom of the heap, i.e., region 0, is pointed to by - // _in_cset_fast_test_base. The _in_cset_fast_test field has been - // biased so that it actually points to address 0 of the address - // space, to make the test as fast as possible (we can simply shift - // the address to address into it, instead of having to subtract the - // bottom of the heap from the address before shifting it; basically - // it works in the same way the card table works). - bool* _in_cset_fast_test; + // This array is used for a quick test on whether a reference points into + // the collection set or not. Each of the array's elements denotes whether the + // corresponding region is in the collection set or not. + G1FastCSetBiasedMappedArray _in_cset_fast_test; - // The allocated array used for the fast test on whether a reference - // points into the collection set or not. This field is also used to - // free the array. - bool* _in_cset_fast_test_base; + class BoolBiasedMappedArray : public G1BiasedMappedArray { + protected: + bool default_value() const { return false; } + public: + void clear() { G1BiasedMappedArray::clear(); } + }; - // The length of the _in_cset_fast_test_base array. - uint _in_cset_fast_test_length; + BoolBiasedMappedArray _humongous_is_live; volatile unsigned _gc_time_stamp; @@ -504,7 +525,7 @@ // attempt to expand the heap if necessary to satisfy the allocation // request. If the region is to be used as an old region or for a // humongous object, set is_old to true. If not, to false. - HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand); + HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand, bool use_partially_free_regions); // Attempt to satisfy a humongous allocation request of the given // size by finding a contiguous set of free regions of num_regions @@ -677,6 +698,8 @@ public: + void check_partial(); + G1MonitoringSupport* g1mm() { assert(_g1mm != NULL, "should have been initialized"); return _g1mm; @@ -692,27 +715,28 @@ virtual void gc_prologue(bool full); virtual void gc_epilogue(bool full); + void set_humongous_is_live(HeapWord* addr) { + if (!_humongous_is_live.get_by_address(addr)) { + _humongous_is_live.set_by_address(addr, true); + } + } + + bool humongous_is_live(uint region) { + return _humongous_is_live.get_by_index(region); + } + // We register a region with the fast "in collection set" test. We // simply set to true the array slot corresponding to this region. void register_region_with_in_cset_fast_test(HeapRegion* r) { - assert(_in_cset_fast_test_base != NULL, "sanity"); assert(r->in_collection_set(), "invariant"); - uint index = r->hrs_index(); - assert(index < _in_cset_fast_test_length, "invariant"); - assert(!_in_cset_fast_test_base[index], "invariant"); - _in_cset_fast_test_base[index] = true; + _in_cset_fast_test.set_in_cset(r->hrs_index()); } // This is a fast test on whether a reference points into the // collection set or not. Assume that the reference // points into the heap. - bool in_cset_fast_test(oop obj) { - assert(_in_cset_fast_test != NULL, "sanity"); - assert(_g1_committed.contains((HeapWord*) obj), err_msg("Given reference outside of heap, is "PTR_FORMAT, (HeapWord*)obj)); - // no need to subtract the bottom of the heap from obj, - // _in_cset_fast_test is biased - uintx index = cast_from_oop(obj) >> HeapRegion::LogOfHRGrainBytes; - bool ret = _in_cset_fast_test[index]; + bool is_in_cset(oop obj) { + bool ret = _in_cset_fast_test.is_in_cset((HeapWord*)obj); // let's make sure the result is consistent with what the slower // test returns assert( ret || !obj_in_cs(obj), "sanity"); @@ -721,9 +745,7 @@ } void clear_cset_fast_test() { - assert(_in_cset_fast_test_base != NULL, "sanity"); - memset(_in_cset_fast_test_base, false, - (size_t) _in_cset_fast_test_length * sizeof(bool)); + _in_cset_fast_test.clear(); } // This is called at the start of either a concurrent cycle or a Full @@ -778,7 +800,7 @@ // list later). The used bytes of freed regions are accumulated in // pre_used. If par is true, the region's RSet will not be freed // up. The assumption is that this will be done later. - void free_humongous_region(HeapRegion* hr, + HeapRegion* free_humongous_region(HeapRegion* hr, FreeRegionList* free_list, bool par); protected: @@ -857,7 +879,8 @@ OopClosure* scan_non_heap_roots, OopsInHeapRegionClosure* scan_rs, G1KlassScanClosure* scan_klasses, - int worker_i); + int worker_i, + GCPhaseTimeTracker* phase_tracker); // Notifies all the necessary spaces that the committed space has // been updated (either expanded or shrunk). It should be called @@ -1051,7 +1074,8 @@ unsigned int* _worker_cset_start_region_time_stamp; enum G1H_process_strong_roots_tasks { - G1H_PS_filter_satb_buffers, + G1H_PS_First, + G1H_PS_filter_satb_buffers = G1H_PS_First, G1H_PS_refProcessor_oops_do, // Leave this one last. G1H_PS_NumElements @@ -1062,8 +1086,17 @@ volatile bool _free_regions_coming; public: +#define HUMONGOUS_FREED_NUM_ENTRIES (64) + size_t _humongous_freed_any_gc_stats[HUMONGOUS_FREED_NUM_ENTRIES + 1]; + size_t _humongous_freed_full_gc_stats[HUMONGOUS_FREED_NUM_ENTRIES + 1]; + size_t _humongous_object_type[2]; // either objArray or type array + + void update_freed_humongous_region_stats(HeapRegion* hr); + void print_freed_humongous_region_stats(outputStream* out = gclog_or_tty); SubTasksDone* process_strong_tasks() { return _process_strong_tasks; } + + static uint num_ext_root_tasks() { return G1H_PS_NumElements + SharedHeap::SH_PS_NumElements; } void set_refine_cte_cl_concurrency(bool concurrent); @@ -1291,6 +1324,10 @@ // set. inline bool obj_in_cs(oop obj); + bool is_in_cset_or_humongous(const oop obj) { + return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj); + } + // Return "TRUE" iff the given object address is in the reserved // region of g1. bool is_in_g1_reserved(const void* p) const { @@ -1473,8 +1510,10 @@ } bool is_in_young(const oop obj) { - HeapRegion* hr = heap_region_containing(obj); - return hr != NULL && hr->is_young(); + if (obj == NULL) { + return false; + } + return heap_region_containing(obj)->is_young(); } #ifdef ASSERT @@ -1582,24 +1621,18 @@ // This will find the region to which the object belongs and // then call the region version of the same function. - // Added if it is NULL it isn't dead. - bool is_obj_dead(const oop obj) const { - const HeapRegion* hr = heap_region_containing(obj); - if (hr == NULL) { - if (obj == NULL) return false; - else return true; - } - else return is_obj_dead(obj, hr); + if (obj == NULL) { + return false; + } + return is_obj_dead(obj, heap_region_containing(obj)); } bool is_obj_ill(const oop obj) const { - const HeapRegion* hr = heap_region_containing(obj); - if (hr == NULL) { - if (obj == NULL) return false; - else return true; + if (obj == NULL) { + return false; } - else return is_obj_ill(obj, hr); + return is_obj_ill(obj, heap_region_containing(obj)); } bool allocated_since_marking(oop obj, HeapRegion* hr, VerifyOption vo); @@ -1863,6 +1896,9 @@ HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) { HeapWord* obj = NULL; + if (_g1h->isHumongous(word_sz)) { + return NULL; + } size_t gclab_word_size = _g1h->desired_plab_sz(purpose); if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose); @@ -2042,15 +2078,16 @@ // set, due to (benign) races in the claim mechanism during RSet scanning more // than one thread might claim the same card. So the same card may be // processed multiple times. So redo this check. - if (_g1h->in_cset_fast_test(obj)) { + if (_g1h->is_in_cset_or_humongous(obj)) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); } else { forwardee = copy_to_survivor_space(obj); } - assert(forwardee != NULL, "forwardee should not be NULL"); - oopDesc::encode_store_heap_oop(p, forwardee); + if (forwardee != NULL) { + oopDesc::encode_store_heap_oop(p, forwardee); + } } assert(obj != NULL, "Must be"); diff --git a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp @@ -39,10 +39,12 @@ template inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { + assert(addr != NULL, "invariant"); HeapRegion* hr = _hrs.addr_to_region((HeapWord*) addr); - // hr can be null if addr in perm_gen - if (hr != NULL && hr->continuesHumongous()) { - hr = hr->humongous_start_region(); + if (hr->continuesHumongous()) { + if (hr->humongous_start_region()->top() > (HeapWord*) addr) { + return hr->humongous_start_region(); + } } return hr; } diff --git a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp @@ -203,7 +203,7 @@ _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime()); _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0; - _phase_times = new G1GCPhaseTimes(_parallel_gc_threads); + _phase_times = NULL; int index = MIN2(_parallel_gc_threads - 1, 7); @@ -439,6 +439,7 @@ } void G1CollectorPolicy::init() { + _phase_times = new G1GCPhaseTimes(_parallel_gc_threads, G1Log::finest() ? G1CollectedHeap::num_ext_root_tasks() : 0); // Set aside an initial future to_space. _g1 = G1CollectedHeap::heap(); @@ -958,8 +959,8 @@ "otherwise, the subtraction below does not make sense"); size_t rs_size = _cur_collection_pause_used_regions_at_start - cset_region_length(); - size_t cur_used_bytes = _g1->used(); - assert(cur_used_bytes == _g1->recalculate_used(), "It should!"); + assert(_g1->used() == _g1->recalculate_used(), err_msg("Used mismatch: used: " SIZE_FORMAT " recalculated: " SIZE_FORMAT, + _g1->used(), _g1->recalculate_used())); bool last_pause_included_initial_mark = false; bool update_stats = !_g1->evacuation_failed(); @@ -1125,7 +1126,7 @@ } _rs_length_diff_seq->add((double) rs_length_diff); - size_t freed_bytes = _heap_used_bytes_before_gc - cur_used_bytes; + size_t freed_bytes = _heap_used_bytes_before_gc - _g1->used(); size_t copied_bytes = _collection_set_bytes_used_before - freed_bytes; double cost_per_byte_ms = 0.0; diff --git a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp @@ -154,10 +154,12 @@ #endif -G1GCPhaseTimes::G1GCPhaseTimes(uint max_gc_threads) : +G1GCPhaseTimes::G1GCPhaseTimes(uint max_gc_threads, uint num_ext_root_scan_phases) : _max_gc_threads(max_gc_threads), + _num_ext_root_scan_phases(num_ext_root_scan_phases), _last_gc_worker_start_times_ms(_max_gc_threads, "%.1lf", false), _last_ext_root_scan_times_ms(_max_gc_threads, "%.1lf"), + _last_ext_root_scan_phase_times_ms(NULL), _last_satb_filtering_times_ms(_max_gc_threads, "%.1lf"), _last_update_rs_times_ms(_max_gc_threads, "%.1lf"), _last_update_rs_processed_buffers(_max_gc_threads, "%d"), @@ -174,6 +176,21 @@ _cur_string_dedup_table_fixup_worker_times_ms(_max_gc_threads, "%.1lf") { assert(max_gc_threads > 0, "Must have some GC threads"); + if (track_ext_root_scan_phases()) { + _last_ext_root_scan_phase_times_ms = NEW_C_HEAP_ARRAY(WorkerDataArray*, num_ext_root_scan_phases, mtGC); + for (uint i = 0; i < num_ext_root_scan_phases; i++) { + _last_ext_root_scan_phase_times_ms[i] = new WorkerDataArray(_max_gc_threads, "%.1lf"); + } + } +} + +G1GCPhaseTimes::~G1GCPhaseTimes() { + if (track_ext_root_scan_phases()) { + for (uint i = 0; i < _num_ext_root_scan_phases; i++) { + delete _last_ext_root_scan_phase_times_ms[i]; + } + FREE_C_HEAP_ARRAY(WorkerDataArray*, _last_ext_root_scan_phase_times_ms, mtGC); + } } void G1GCPhaseTimes::note_gc_start(uint active_gc_threads) { @@ -195,6 +212,10 @@ _last_gc_worker_end_times_ms.reset(); _last_gc_worker_times_ms.reset(); _last_gc_worker_other_times_ms.reset(); + + for (uint i = 0; i < _num_ext_root_scan_phases; i++) { + _last_ext_root_scan_phase_times_ms[i]->reset(); + } } void G1GCPhaseTimes::note_gc_end() { @@ -286,6 +307,18 @@ print_stats(1, "Parallel Time", _cur_collection_par_time_ms, _active_gc_threads); _last_gc_worker_start_times_ms.print(2, "GC Worker Start (ms)"); _last_ext_root_scan_times_ms.print(2, "Ext Root Scanning (ms)"); + if (track_ext_root_scan_phases()) { + const char *headers[] = { + "SH_PS_Universe_oops_do", "SH_PS_JNIHandles_oops_do", "SH_PS_ObjectSynchronizer_oops_do", + "SH_PS_FlatProfiler_oops_do", "SH_PS_Management_oops_do", "SH_PS_SystemDictionary_oops_do", + "SH_PS_ClassLoaderDataGraph_oops_do", "SH_PS_jvmti_oops_do", "SH_PS_CodeCache_oops_do", + "SH_PS_Threads_oops_do", "SH_PS_StringTable_oops_do", + "G1H_PS_filter_satb_buffers", "G1H_PS_refProcessor_oops_do" + }; + for (uint i = 0; i < _num_ext_root_scan_phases; i++) { + _last_ext_root_scan_phase_times_ms[i]->print(3, headers[i]); + } + } if (_last_satb_filtering_times_ms.sum() > 0.0) { _last_satb_filtering_times_ms.print(2, "SATB Filtering (ms)"); } @@ -350,6 +383,9 @@ if (G1DeferredRSUpdate) { print_stats(2, "Redirty Cards", _recorded_redirty_logged_cards_time_ms); } + if (G1ReclaimDeadLargeObjectsAtYoungGC) { + print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms); + } print_stats(2, "Free CSet", (_recorded_young_free_cset_time_ms + _recorded_non_young_free_cset_time_ms)); diff --git a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp @@ -107,14 +107,37 @@ } }; +class GCPhaseTimeTracker VALUE_OBJ_CLASS_SPEC { +private: + friend class G1GCPhaseTimes; + + WorkerDataArray** _data; + uint _num_phases; + uint _worker_id; +public: + GCPhaseTimeTracker(WorkerDataArray** data, uint num_phases, uint worker_id) : + _data(data), _num_phases(num_phases), _worker_id(worker_id) { } + + bool active() const { return _num_phases > 0; } + + void set_value(uint phase, double value) { + assert(_data != NULL, "just checking"); + _data[phase]->set(_worker_id, value); + } +}; + class G1GCPhaseTimes : public CHeapObj { private: uint _active_gc_threads; uint _max_gc_threads; + uint _num_ext_root_scan_phases; + + bool track_ext_root_scan_phases() const { return _num_ext_root_scan_phases > 0; } WorkerDataArray _last_gc_worker_start_times_ms; WorkerDataArray _last_ext_root_scan_times_ms; + WorkerDataArray** _last_ext_root_scan_phase_times_ms; WorkerDataArray _last_satb_filtering_times_ms; WorkerDataArray _last_update_rs_times_ms; WorkerDataArray _last_update_rs_processed_buffers; @@ -156,6 +179,8 @@ double _recorded_young_free_cset_time_ms; double _recorded_non_young_free_cset_time_ms; + double _cur_fast_reclaim_humongous_time_ms; + double _cur_verify_before_time_ms; double _cur_verify_after_time_ms; @@ -164,7 +189,12 @@ void print_stats(int level, const char* str, double value, int workers); public: - G1GCPhaseTimes(uint max_gc_threads); + G1GCPhaseTimes(uint max_gc_threads, uint num_ext_root_scan_phases); + virtual ~G1GCPhaseTimes(); + + WorkerDataArray** get_ext_root_scan_phase_times() const { return _last_ext_root_scan_phase_times_ms; } + uint num_ext_root_scan_phases() const { return _num_ext_root_scan_phases; } + void note_gc_start(uint active_gc_threads); void note_gc_end(); void print(double pause_time_sec); @@ -285,6 +315,8 @@ _recorded_non_young_free_cset_time_ms = time_ms; } + double* fast_reclaim_humongous_time_ms_addr() { return &_cur_fast_reclaim_humongous_time_ms; } + void record_young_cset_choice_time_ms(double time_ms) { _recorded_young_cset_choice_time_ms = time_ms; } @@ -343,6 +375,10 @@ return _recorded_non_young_free_cset_time_ms; } + double fast_reclaim_humongous_time_ms() { + return _cur_fast_reclaim_humongous_time_ms; + } + double average_last_update_rs_time() { return _last_update_rs_times_ms.average(); } diff --git a/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp b/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp --- a/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp +++ b/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp @@ -199,7 +199,7 @@ void free_humongous_region(HeapRegion* hr) { HeapWord* end = hr->end(); - FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep"); + FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep", false, true); assert(hr->startsHumongous(), "Only the start of a humongous region should be freed."); @@ -207,8 +207,18 @@ hr->set_containing_set(NULL); _humongous_regions_removed.increment(1u, hr->capacity()); - _g1h->free_humongous_region(hr, &dummy_free_list, false /* par */); - hr->prepare_for_compaction(&_cp); + HeapRegion* last_region = _g1h->free_humongous_region(hr, &dummy_free_list, false /* par */); + if (last_region != NULL) { + if (last_region->used_by_own_objects() == 0) { + last_region->set_first_object_start(last_region->bottom()); + last_region->reset_non_humongous_space(); + _g1h->free_region(last_region, &dummy_free_list, false); + } else { + last_region->make_old(); + } + } + _g1h->check_partial(); + hr->prepare_for_compaction(&_cp); // TODO: why do we only prepare the first region for compaction? // Also clear the part of the card table that will be unused after // compaction. _mrbs->clear(MemRegion(hr->compaction_top(), end)); @@ -240,6 +250,11 @@ } } else { assert(hr->continuesHumongous(), "Invalid humongous."); + if (hr->has_own_objects()) { + // handle the objects at the end of the humongous object + hr->non_humongous_space()->prepare_for_compaction(&_cp); + _mrbs->clear(MemRegion(hr->humongous_start_region()->top(), hr->end())); + } } } else { hr->prepare_for_compaction(&_cp); @@ -277,17 +292,23 @@ class G1AdjustPointersClosure: public HeapRegionClosure { public: - bool doHeapRegion(HeapRegion* r) { - if (r->isHumongous()) { - if (r->startsHumongous()) { + bool doHeapRegion(HeapRegion* hr) { + if (hr->isHumongous()) { + if (hr->startsHumongous()) { // We must adjust the pointers on the single H object. - oop obj = oop(r->bottom()); + oop obj = oop(hr->bottom()); // point all the oops to the new location obj->adjust_pointers(); + } else { + assert(hr->continuesHumongous(), "Invalid humongous."); + if (hr->has_own_objects()) { + // handle the objects at the end of the humongous object + hr->non_humongous_space()->adjust_pointers(); + } } } else { // This really ought to be "as_CompactibleSpace"... - r->adjust_pointers(); + hr->adjust_pointers(); } return false; } @@ -341,6 +362,15 @@ assert(hr->is_empty(), "Should have been cleared in phase 2."); } hr->reset_during_compaction(); + } else if (hr->has_own_objects()) { + hr->non_humongous_space()->compact(); + //hr->set_top(hr->humongous_start_region()->top()); +#ifdef ASSERT + if (hr->containing_set() != NULL) { + hr->set_containing_set(NULL); + } +#endif + hr->reset_non_humongous_space(); } } else { hr->compact(); diff --git a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp @@ -41,7 +41,7 @@ inline void FilterIntoCSClosure::do_oop_nv(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop) && - _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) { + _g1->is_in_cset_or_humongous(oopDesc::decode_heap_oop_not_null(heap_oop))) { _oc->do_oop(p); } } @@ -64,7 +64,7 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->in_cset_fast_test(obj)) { + if (_g1->is_in_cset_or_humongous(obj)) { // We're not going to even bother checking whether the object is // already forwarded or not, as this usually causes an immediate // stall. We'll try to prefetch the object (for write, given that @@ -94,18 +94,21 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->in_cset_fast_test(obj)) { + if (_g1->is_in_cset_or_humongous(obj)) { Prefetch::write(obj->mark_addr(), 0); Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); // Place on the references queue _par_scan_state->push_on_queue(p); + } else { + assert(!_g1->is_in_cset(obj), "checking"); } } } template inline void G1CMOopClosure::do_oop_nv(T* p) { + assert(p != NULL, "invariant"); assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); assert(!_g1h->is_on_master_free_list( _g1h->heap_region_containing((HeapWord*) p)), "invariant"); @@ -125,9 +128,7 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); HeapRegion* hr = _g1h->heap_region_containing((HeapWord*) obj); - if (hr != NULL) { - _cm->grayRoot(obj, obj->size(), _worker_id, hr); - } + _cm->grayRoot(obj, obj->size(), _worker_id, hr); } } @@ -154,27 +155,28 @@ template inline void G1UpdateRSOrPushRefOopClosure::do_oop_nv(T* p) { oop obj = oopDesc::load_decode_heap_oop(p); + if (obj == NULL) { + return; + } #ifdef ASSERT // can't do because of races // assert(obj == NULL || obj->is_oop(), "expected an oop"); // Do the safe subset of is_oop - if (obj != NULL) { #ifdef CHECK_UNHANDLED_OOPS - oopDesc* o = obj.obj(); + oopDesc* o = obj.obj(); #else - oopDesc* o = obj; + oopDesc* o = obj; #endif // CHECK_UNHANDLED_OOPS - assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); - assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); - } + assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); + assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); #endif // ASSERT assert(_from != NULL, "from region must be non-NULL"); assert(_from->is_in_reserved(p), "p is not in from"); HeapRegion* to = _g1->heap_region_containing(obj); - if (to != NULL && _from != to) { + if (_from != to) { // The _record_refs_into_cset flag is true during the RSet // updating part of an evacuation pause. It is false at all // other times: diff --git a/src/share/vm/gc_implementation/g1/g1RemSet.cpp b/src/share/vm/gc_implementation/g1/g1RemSet.cpp --- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp @@ -208,8 +208,8 @@ card_start, card_start + CardTableModRefBS::card_size_in_words); #endif + assert(card_start != NULL, "invariant"); HeapRegion* card_region = _g1h->heap_region_containing(card_start); - assert(card_region != NULL, "Yielding cards not in the heap?"); _cards++; if (!card_region->is_on_dirty_cards_region_list()) { @@ -403,8 +403,8 @@ // Construct the region representing the card. HeapWord* start = _ct_bs->addr_for(card_ptr); // And find the region containing it. + assert(start != NULL, "invariant"); HeapRegion* r = _g1->heap_region_containing(start); - assert(r != NULL, "unexpected null"); // Scan oops in the card looking for references into the collection set // Don't use addr_for(card_ptr + 1) which can ask for @@ -507,7 +507,7 @@ _ctbs(_g1h->g1_barrier_set()) {} bool doHeapRegion(HeapRegion* r) { - if (!r->continuesHumongous()) { + if (r->has_own_objects()) { r->rem_set()->scrub(_ctbs, _region_bm, _card_bm); } return false; @@ -564,13 +564,10 @@ // Construct the region representing the card. HeapWord* start = _ct_bs->addr_for(card_ptr); + HeapWord* end = start + CardTableModRefBS::card_size_in_words; // And find the region containing it. + assert(start != NULL, "invariant"); HeapRegion* r = _g1->heap_region_containing(start); - if (r == NULL) { - // Again no need to return that this card contains refs that - // point into the collection set. - return false; // Not in the G1 heap (might be in perm, for example.) - } // Why do we have to check here whether a card is on a young region, // given that we dirty young regions and, as a result, the @@ -622,11 +619,9 @@ } start = _ct_bs->addr_for(card_ptr); + end = start + CardTableModRefBS::card_size_in_words; + assert(start != NULL, "invariant"); r = _g1->heap_region_containing(start); - if (r == NULL) { - // Not in the G1 heap - return false; - } // Checking whether the region we got back from the cache // is young here is inappropriate. The region could have been @@ -637,8 +632,7 @@ // Don't use addr_for(card_ptr + 1) which can ask for // a card beyond the heap. This is not safe without a perm // gen at the upper end of the heap. - HeapWord* end = start + CardTableModRefBS::card_size_in_words; - MemRegion dirtyRegion(start, end); + MemRegion dirtyRegion(MAX2(start, r->first_object_start()), end); #if CARD_REPEAT_HISTO init_ct_freq_table(_g1->max_capacity()); diff --git a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp +++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp @@ -63,10 +63,12 @@ assert(from == NULL || from->is_in_reserved(p), "p is not in from"); - HeapRegion* to = _g1->heap_region_containing(obj); - if (to != NULL && from != to) { - assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); - to->rem_set()->add_reference(p, tid); + if (obj != NULL) { + HeapRegion* to = _g1->heap_region_containing(obj); + if (from != to) { + assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); + to->rem_set()->add_reference(p, tid); + } } } diff --git a/src/share/vm/gc_implementation/g1/g1_globals.hpp b/src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp @@ -108,6 +108,9 @@ develop(bool, G1RSBarrierRegionFilter, true, \ "If true, generate region filtering code in RS barrier") \ \ + experimental(bool, G1TraceHumongousObjectLiveness, false, \ + "Trace information about liveness of humongous objects.") \ + \ develop(bool, G1DeferredRSUpdate, true, \ "If true, use deferred RS updates") \ \ @@ -289,6 +292,13 @@ "The amount of code root chunks that should be kept at most " \ "as percentage of already allocated.") \ \ + experimental(bool, G1ReclaimDeadLargeObjectsAtYoungGC, true, \ + "Try to reclaim dead large objects at every young GC.") \ + \ + experimental(bool, G1TraceReclaimDeadLargeObjectsAtYoungGC, false, \ + "Print some information about large object liveness " \ + "at every young GC.") \ + \ experimental(uintx, G1OldCSetRegionThresholdPercent, 10, \ "An upper bound for the number of old CSet regions expressed " \ "as a percentage of the heap size.") \ @@ -329,7 +339,14 @@ "remembered set when verifying the heap during a full GC.") \ \ diagnostic(bool, G1VerifyHeapRegionCodeRoots, false, \ - "Verify the code root lists attached to each heap region.") + "Verify the code root lists attached to each heap region.") \ + \ + product(bool, G1AllocObjsInHCRegions, true, \ + "Allow allocation of objects in last continues humongous " \ + "regions") \ + \ + product(bool, G1VerboseAllocObjsInHCRegions, false, \ + "Enable extra logging for HC alloc regions") G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) diff --git a/src/share/vm/gc_implementation/g1/heapRegion.cpp b/src/share/vm/gc_implementation/g1/heapRegion.cpp --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp @@ -352,7 +352,7 @@ _hrs_index(hrs_index), _humongous_type(NotHumongous), _humongous_start_region(NULL), _in_collection_set(false), - _next_in_special_set(NULL), _orig_end(NULL), + _next_in_special_set(NULL), _orig_end(NULL), _first_object_start(_bottom), _claimed(InitialClaimValue), _evacuation_failed(false), _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0), _young_type(NotYoung), _next_young_region(NULL), @@ -362,7 +362,8 @@ #endif // ASSERT _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), - _predicted_bytes_to_copy(0) + _predicted_bytes_to_copy(0), _humongous_allocated_at_gc(0), _humongous_allocated_at_full_gc(0), + _non_humongous_space(NULL) { _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); _orig_end = mr.end(); @@ -384,12 +385,50 @@ HeapRegion* hr = g1h->region_at(index); if (!hr->isHumongous()) { return hr; + } else if (!hr->startsHumongous() && hr->has_own_objects()) { + return hr->non_humongous_space(); } index += 1; } return NULL; } +void HeapRegion::make_old() { + assert(!isHumongous(), "Should have been reset already"); + assert(first_object_start() > bottom(), "Don't call this otherwise"); + + if (G1VerboseAllocObjsInHCRegions) { + gclog_or_tty->print_cr("Turning %u into an old region.", hrs_index()); + } + size_t dummy_obj_size = pointer_delta(first_object_start(), bottom()); + CollectedHeap::fill_with_object(bottom(), dummy_obj_size); + if (G1VerboseAllocObjsInHCRegions) { + gclog_or_tty->print_cr("Clearing: " PTR_FORMAT " - " PTR_FORMAT, bottom(), first_object_start()); + } + + assert(!is_empty(), "empty"); + reset_bot(); + HeapWord* obj_start = bottom(); + size_t obj_size = dummy_obj_size; + HeapWord* obj_end = obj_start + obj_size; + assert(obj_end <= top(), "Empty regions should not get here."); + while (obj_end < top()) { + update_bot_for_object(obj_start, obj_size); + _offsets.verify_for_object(obj_start, obj_size); + obj_start = obj_end; + oop obj = (oop)obj_start; + obj_size = obj->size(); + obj_end = obj_start + obj_size; + } + assert(obj_end == top(), "The last object should end at top"); + assert(!is_empty(), "empty"); + update_bot_for_object(obj_start, obj_size); + + reset_non_humongous_space(); + + set_first_object_start(bottom()); +} + void HeapRegion::save_marks() { set_saved_mark(); } @@ -728,7 +767,8 @@ return; } - if (continuesHumongous()) { + //if (continuesHumongous()) { // TODO: need to allow C-hum regions with objects here? + if (!has_own_objects()) { if (strong_code_roots_length > 0) { gclog_or_tty->print_cr("region "HR_FORMAT" is a continuation of a humongous " "region but has "SIZE_FORMAT" code root entries", @@ -748,9 +788,12 @@ void HeapRegion::print() const { print_on(gclog_or_tty); } void HeapRegion::print_on(outputStream* st) const { + st->print("%3u ", hrs_index()); if (isHumongous()) { if (startsHumongous()) st->print(" HS"); + else if (has_own_objects()) + st->print(" HO"); else st->print(" HC"); } else { @@ -833,6 +876,7 @@ gclog_or_tty->print_cr("----------"); } if (!_g1h->is_in_closed_subset(obj)) { + assert((HeapWord*)p != NULL, "invariant"); HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); gclog_or_tty->print_cr("Field "PTR_FORMAT " of live obj "PTR_FORMAT" in region " @@ -843,17 +887,19 @@ gclog_or_tty->print_cr("points to obj "PTR_FORMAT" not in the heap", (void*) obj); } else { + assert((HeapWord*)p != NULL, "invariant"); + assert((HeapWord*)obj != NULL, "invariant"); HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj); gclog_or_tty->print_cr("Field "PTR_FORMAT - " of live obj "PTR_FORMAT" in region " + " of live obj "PTR_FORMAT" in region %u " "["PTR_FORMAT", "PTR_FORMAT")", - p, (void*) _containing_obj, + p, (void*) _containing_obj, from->hrs_index(), from->bottom(), from->end()); print_object(gclog_or_tty, _containing_obj); - gclog_or_tty->print_cr("points to dead obj "PTR_FORMAT" in region " + gclog_or_tty->print_cr("points to dead obj "PTR_FORMAT" in region %u" "["PTR_FORMAT", "PTR_FORMAT")", - (void*) obj, to->bottom(), to->end()); + (void*) obj, to->hrs_index(), to->bottom(), to->end()); print_object(gclog_or_tty, obj); } gclog_or_tty->print_cr("----------"); @@ -861,14 +907,17 @@ _failures = true; failed = true; _n_failures++; + assert(false, "dead obj"); } if (!_g1h->full_collection() || G1VerifyRSetsDuringFullGC) { + assert((HeapWord*)p != NULL, "invariant"); + assert((HeapWord*)obj != NULL, "invariant"); HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); HeapRegion* to = _g1h->heap_region_containing(obj); if (from != NULL && to != NULL && from != to && - !to->isHumongous()) { + !to->isHumongous() /* TODO: can we check c-hum regions with objects? */) { jbyte cv_obj = *_bs->byte_for_const(_containing_obj); jbyte cv_field = *_bs->byte_for_const(p); const jbyte dirty = CardTableModRefBS::dirty_card_val(); @@ -923,14 +972,24 @@ HeapWord* prev_p = NULL; VerifyLiveClosure vl_cl(g1, vo); bool is_humongous = isHumongous(); + bool is_mixed_humongous = is_humongous && has_own_objects(); bool do_bot_verify = !is_young(); + + if (is_mixed_humongous) { + p = first_object_start(); + if (top() == p) { + // there are no objects in this region yet. The BOT verification below will fail. + return; + } + } + size_t object_num = 0; while (p < top()) { oop obj = oop(p); size_t obj_size = obj->size(); object_num += 1; - if (is_humongous != g1->isHumongous(obj_size)) { + if (!is_mixed_humongous && is_humongous != g1->isHumongous(obj_size)) { gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" SIZE_FORMAT" words) in a %shumongous region", p, g1->isHumongous(obj_size) ? "" : "non-", @@ -1043,7 +1102,7 @@ } } - if (is_humongous && object_num > 1) { + if (!is_mixed_humongous && is_humongous && object_num > 1) { gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " "but has "SIZE_FORMAT", objects", bottom(), end(), object_num); @@ -1063,8 +1122,8 @@ // away eventually. void G1OffsetTableContigSpace::clear(bool mangle_space) { + _offsets.zero_bottom_entry(); ContiguousSpace::clear(mangle_space); - _offsets.zero_bottom_entry(); _offsets.initialize_threshold(); } @@ -1089,6 +1148,10 @@ return _offsets.initialize_threshold(); } +HeapWord* G1OffsetTableContigSpace::initialize_threshold(HeapWord* bottom) { + return _offsets.initialize_threshold(bottom); +} + HeapWord* G1OffsetTableContigSpace::cross_threshold(HeapWord* start, HeapWord* end) { _offsets.alloc_block(start, end); @@ -1140,3 +1203,15 @@ _offsets.zero_bottom_entry(); _offsets.initialize_threshold(); } + + +CompactibleSpace* SubHeapRegion::next_compaction_space() const { return _hr->next_compaction_space(); } +HeapWord* SubHeapRegion::initialize_threshold() { return _hr->initialize_threshold(_bottom); } +HeapWord* SubHeapRegion::cross_threshold(HeapWord* start, HeapWord* end) {return _hr->cross_threshold(start, end); } +void SubHeapRegion::set_top(HeapWord* top) { _hr->set_top(top); } +HeapWord* SubHeapRegion::top() const { return _hr->top(); } +HeapWord* SubHeapRegion::compaction_top() const { return _hr->compaction_top(); } +void SubHeapRegion::set_compaction_top(HeapWord* value) { _hr->set_compaction_top(value); } +void SubHeapRegion::reset_after_compaction() { _hr->reset_after_compaction(); }; +MemRegion SubHeapRegion::used_region() const { return _hr->used_region(); } +void SubHeapRegion::clear(bool mangle_space) { _hr->clear(mangle_space); } diff --git a/src/share/vm/gc_implementation/g1/heapRegion.hpp b/src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp @@ -155,7 +155,9 @@ class G1OffsetTableContigSpace: public ContiguousSpace { friend class VMStructs; protected: + public: G1BlockOffsetArrayContigSpace _offsets; + protected: Mutex _par_alloc_lock; volatile unsigned _gc_time_stamp; // When we need to retire an allocation region, while other threads @@ -200,6 +202,7 @@ // MarkSweep support phase3 virtual HeapWord* initialize_threshold(); + virtual HeapWord* initialize_threshold(HeapWord* bottom); virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); virtual void print() const; @@ -218,6 +221,36 @@ } }; +class SubHeapRegion : public ContiguousSpace { + HeapRegion* _hr; +public: + SubHeapRegion(HeapRegion* hr) : _hr(hr) { } + + CompactibleSpace* next_compaction_space() const; + HeapWord* initialize_threshold(); + HeapWord* cross_threshold(HeapWord* start, HeapWord* end); + + HeapWord* block_start_const(const void* p) const { + ShouldNotReachHere(); + return NULL; + } + //void clear(bool mangle_space) { ShouldNotReachHere(); } + virtual HeapWord* block_start(const void* p) { + ShouldNotReachHere(); + return NULL; + } + + void set_top(HeapWord* top); + HeapWord* top() const; + + HeapWord* compaction_top() const; + void set_compaction_top(HeapWord* value); + void reset_after_compaction(); + + MemRegion used_region() const; + void clear(bool mangle_space); +}; + class HeapRegion: public G1OffsetTableContigSpace { friend class VMStructs; private: @@ -239,6 +272,8 @@ G1BlockOffsetArrayContigSpace* offsets() { return &_offsets; } + ContiguousSpace* _non_humongous_space; + protected: // The index of this region in the heap region sequence. uint _hrs_index; @@ -249,6 +284,8 @@ // For the start region of a humongous sequence, it's original end(). HeapWord* _orig_end; + HeapWord* _first_object_start; + // True iff the region is in current collection_set. bool _in_collection_set; @@ -339,7 +376,18 @@ // the total value for the collection set. size_t _predicted_bytes_to_copy; + // The time (in GCs) when this humongous region has been allocated + unsigned int _humongous_allocated_at_gc; + unsigned int _humongous_allocated_at_full_gc; public: + void set_humongous_allocated_at(uint allocated_at_gc, uint allocated_at_full_gc) { + _humongous_allocated_at_gc = allocated_at_gc; + _humongous_allocated_at_full_gc = allocated_at_full_gc; + } + + unsigned int humongous_allocated_at_gc() const { return _humongous_allocated_at_gc; } + unsigned int humongous_allocated_at_full_gc() const { return _humongous_allocated_at_full_gc; } + HeapRegion(uint hrs_index, G1BlockOffsetSharedArray* sharedOffsetArray, MemRegion mr); @@ -408,8 +456,11 @@ // A lower bound on the amount of garbage bytes in the region. size_t garbage_bytes() { - size_t used_at_mark_start_bytes = - (prev_top_at_mark_start() - bottom()) * HeapWordSize; + size_t used_at_mark_start_bytes = 0; + if (prev_top_at_mark_start() > first_object_start()) { + used_at_mark_start_bytes = (prev_top_at_mark_start() - first_object_start()) * HeapWordSize; + } + //(prev_top_at_mark_start() - bottom()) * HeapWordSize; assert(used_at_mark_start_bytes >= marked_bytes(), "Can't mark more than we have."); return used_at_mark_start_bytes - marked_bytes(); @@ -426,7 +477,12 @@ } // An upper bound on the number of live bytes in the region. - size_t max_live_bytes() { return used() - garbage_bytes(); } + size_t max_live_bytes() { + size_t used_b = used(); + size_t garbage = garbage_bytes(); + assert(used_b >= garbage, err_msg("overflow, used: " SIZE_FORMAT " garbage: " SIZE_FORMAT, used_b, garbage)); + return used_b - garbage; + } void add_to_marked_bytes(size_t incr_bytes) { _next_marked_bytes = _next_marked_bytes + incr_bytes; @@ -540,6 +596,79 @@ void set_next(HeapRegion* next) { _next = next; } void set_prev(HeapRegion* prev) { _prev = prev; } + bool has_own_objects() const { + if (!continuesHumongous()) { + // "Normal" regions and starts humongous regions are considered to have their own objects. + return true; + } + // Continues humongous regions only have their own objects if we have allocated new objects + // at the end of the humongous object. + return _first_object_start != _bottom; + //return top() > _humongous_start_region->top(); + } + + ContiguousSpace* non_humongous_space() { + assert(continuesHumongous() && has_own_objects(), "Don't call this otherwise"); + + if (_non_humongous_space != NULL) { + assert(_non_humongous_space->bottom() == first_object_start(), "Mismatching non-humongous space"); + } else { + MemRegion mr(first_object_start(), end()); + _non_humongous_space = new SubHeapRegion(this); + _non_humongous_space->initialize(mr, false, false); + } + assert(top() >= _non_humongous_space->bottom(), "Must be"); + return _non_humongous_space; + } + + void reset_non_humongous_space() { + if (_non_humongous_space != NULL) { + delete _non_humongous_space; + _non_humongous_space = NULL; + } + } + + void set_first_object_start(HeapWord* first_object_start) { + _first_object_start = first_object_start; + } + + HeapWord* first_object_start() const { + return _first_object_start; + /* + assert(has_own_objects(), "Don't call otherwise"); + if (!continuesHumongous()) { + return bottom(); + } + return (HeapWord*)align_size_up((intptr_t)(_humongous_start_region->top()), CardTableModRefBS::card_size); + //return _humongous_start_region->top(); + */ + } + + void make_old(); + + size_t used_by_own_objects() { + return byte_size(first_object_start(), top()); + } + + size_t used() const { + if (continuesHumongous()) { + if (has_own_objects()) { + // This is a region that has trailing object after the humongous object. + // Report the space they use up as used. + // The start humongous region will report the whole humongous object as used. + return byte_size(_humongous_start_region->top(), top()); + //return byte_size(first_object_start(), top()); + } else { + return 0; + } + } + return G1OffsetTableContigSpace::used(); + } + + bool is_empty() const { + return !continuesHumongous() && used() == 0; + } + // Every region added to a set is tagged with a reference to that // set. This is used for doing consistency checking to make sure that // the contents of a set are as they should be and it's only diff --git a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @@ -821,7 +821,6 @@ bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { HeapRegion* hr = _g1h->heap_region_containing_raw(from); - if (hr == NULL) return false; RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); // Is this region in the coarse map? if (_coarse_map.at(hr_ind)) return true; @@ -987,6 +986,7 @@ _num_self_forwarded++; } else { // The reference points into a promotion or to-space region + assert(obj != NULL, "invariant"); HeapRegion* to = _g1h->heap_region_containing(obj); to->rem_set()->add_strong_code_root(_nm); } diff --git a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp @@ -272,6 +272,13 @@ return _other_regions.hr(); } + bool is_empty() const { + return (strong_code_roots_list_length() == 0) && + (occ_coarse() == 0) && + (occ_sparse() == 0) && + (occ_fine() == 0); + } + size_t occupied() { MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); return occupied_locked(); @@ -378,7 +385,7 @@ void strong_code_roots_do(CodeBlobClosure* blk) const; // Returns the number of elements in the strong code roots list - size_t strong_code_roots_list_length() { + size_t strong_code_roots_list_length() const { return _code_roots.length(); } diff --git a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp @@ -41,7 +41,8 @@ uint first = G1_NULL_HRS_INDEX; uint num_so_far = 0; while (curr < len && num_so_far < num) { - if (at(curr)->is_empty()) { + HeapRegion* hr = at(curr); + if (!hr->continuesHumongous() && hr->is_empty()) { if (first == G1_NULL_HRS_INDEX) { first = curr; num_so_far = 1; diff --git a/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp b/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp @@ -35,12 +35,11 @@ } inline HeapRegion* HeapRegionSeq::addr_to_region(HeapWord* addr) const { - if (addr != NULL && addr < heap_end()) { - assert(addr >= heap_bottom(), - err_msg("addr: "PTR_FORMAT" bottom: "PTR_FORMAT, addr, heap_bottom())); - return addr_to_region_unsafe(addr); - } - return NULL; + assert(addr < heap_end(), + err_msg("addr: "PTR_FORMAT" end: "PTR_FORMAT, addr, heap_end())); + assert(addr >= heap_bottom(), + err_msg("addr: "PTR_FORMAT" bottom: "PTR_FORMAT, addr, heap_bottom())); + return addr_to_region_unsafe(addr); } inline HeapRegion* HeapRegionSeq::at(uint index) const { diff --git a/src/share/vm/gc_implementation/g1/heapRegionSet.cpp b/src/share/vm/gc_implementation/g1/heapRegionSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.cpp +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.cpp @@ -28,6 +28,16 @@ uint FreeRegionList::_unrealistically_long_length = 0; +void HeapRegionSetBase::remove(HeapRegion* hr) { + check_mt_safety(); + verify_region(hr); + assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); + + hr->set_containing_set(NULL); + assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); + _count.decrement(1u, hr->capacity()); +} + void HeapRegionSetBase::fill_in_ext_msg(hrs_ext_msg* msg, const char* message) { msg->append("[%s] %s ln: %u cy: "SIZE_FORMAT, name(), message, length(), total_capacity_bytes()); @@ -38,8 +48,8 @@ void HeapRegionSetBase::verify_region(HeapRegion* hr) { assert(hr->containing_set() == this, err_msg("Inconsistent containing set for %u", hr->hrs_index())); assert(!hr->is_young(), err_msg("Adding young region %u", hr->hrs_index())); // currently we don't use these sets for young regions - assert(hr->isHumongous() == regions_humongous(), err_msg("Wrong humongous state for region %u and set %s", hr->hrs_index(), name())); - assert(hr->is_empty() == regions_empty(), err_msg("Wrong empty state for region %u and set %s", hr->hrs_index(), name())); + //assert(hr->isHumongous() == regions_humongous(), err_msg("Wrong humongous state for region %u and set %s", hr->hrs_index(), name())); + //assert(hr->is_empty() == regions_empty(), err_msg("Wrong empty state for region %u and set %s", hr->hrs_index(), name())); assert(hr->rem_set()->verify_ready_for_par_iteration(), err_msg("Wrong iteration state %u", hr->hrs_index())); } #endif @@ -151,6 +161,45 @@ from_list->verify_optional(); } +void FreeRegionList::remove(HeapRegion* hr) { + check_mt_safety(); + + if (is_empty()) { + return; + } + + if (head() == hr) { + remove_head(); + return; + } + + if (tail() == hr) { + remove_tail(); + return; + } + + HeapRegion* curr = head(); + HeapRegion* prev = curr; + curr = curr->next(); + while (curr != NULL) { + if (curr == hr) { + prev->set_next(curr->next()); + if (curr->next() != NULL) { + curr->next()->set_prev(prev); + curr->set_next(NULL); + } + curr->set_prev(NULL); + HeapRegionSetBase::remove(curr); + if (_last == curr) { + _last = NULL; + } + return; + } + prev = curr; + curr = curr->next(); + } +} + void FreeRegionList::add_as_head(FreeRegionList* from_list) { add_as_head_or_tail(from_list, true /* as_head */); } @@ -287,7 +336,7 @@ curr->set_next(NULL); curr->set_prev(NULL); - remove(curr); + HeapRegionSetBase::remove(curr); curr->set_pending_removal(false); count += 1; diff --git a/src/share/vm/gc_implementation/g1/heapRegionSet.hpp b/src/share/vm/gc_implementation/g1/heapRegionSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.hpp @@ -133,7 +133,7 @@ // It updates the fields of the set to reflect hr being removed // from the set and tags the region appropriately. - inline void remove(HeapRegion* hr); + virtual void remove(HeapRegion* hr); // fill_in_ext_msg() writes the the values of the set's attributes // in the custom err_msg (hrs_ext_msg). fill_in_ext_msg_extra() @@ -225,8 +225,8 @@ virtual void clear(); public: - FreeRegionList(const char* name, HRSMtSafeChecker* mt_safety_checker = NULL): - HeapRegionSetBase(name, false /* humongous */, true /* empty */, mt_safety_checker) { + FreeRegionList(const char* name, bool humongous, bool empty, HRSMtSafeChecker* mt_safety_checker = NULL): + HeapRegionSetBase(name, humongous, empty, mt_safety_checker) { clear(); } @@ -271,6 +271,8 @@ // determined by hrs_index. void add_ordered(FreeRegionList* from_list); + virtual void remove(HeapRegion* hr); + // It moves the regions from from_list to this list and empties // from_list. The new regions will appear in the same order as they // were in from_list and be linked in the beginning of this list. diff --git a/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp b/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp @@ -29,7 +29,7 @@ inline void HeapRegionSetBase::add(HeapRegion* hr) { check_mt_safety(); - assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set %u")); + assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set")); assert(hr->next() == NULL, hrs_ext_msg(this, "should not already be linked")); _count.increment(1u, hr->capacity()); @@ -37,16 +37,6 @@ verify_region(hr); } -inline void HeapRegionSetBase::remove(HeapRegion* hr) { - check_mt_safety(); - verify_region(hr); - assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); - - hr->set_containing_set(NULL); - assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); - _count.decrement(1u, hr->capacity()); -} - inline void FreeRegionList::add_ordered(HeapRegion* hr) { check_mt_safety(); assert((length() == 0 && _head == NULL && _tail == NULL) || @@ -70,6 +60,8 @@ curr = curr->next(); } + guarantee(curr == NULL || curr != hr, err_msg("Adding region %u twice.", hr->hrs_index())); + hr->set_next(curr); if (curr == NULL) { @@ -102,6 +94,8 @@ // add() will verify the region and check mt safety. add(hr); + guarantee(_head != hr, err_msg("Adding region %u twice.", hr->hrs_index())); + // Now link the region. if (_head != NULL) { hr->set_next(_head); @@ -120,6 +114,8 @@ // add() will verify the region and check mt safety. add(hr); + guarantee(_tail != hr, err_msg("Adding region %u twice.", hr->hrs_index())); + // Now link the region. if (_tail != NULL) { _tail->set_next(hr); @@ -150,7 +146,7 @@ } // remove() will verify the region and check mt safety. - remove(hr); + HeapRegionSetBase::remove(hr); return hr; } @@ -184,7 +180,7 @@ } // remove() will verify the region and check mt safety. - remove(hr); + HeapRegionSetBase::remove(hr); return hr; } diff --git a/src/share/vm/memory/referenceProcessor.cpp b/src/share/vm/memory/referenceProcessor.cpp --- a/src/share/vm/memory/referenceProcessor.cpp +++ b/src/share/vm/memory/referenceProcessor.cpp @@ -365,6 +365,7 @@ // Self-loop next, so as to make Ref not active. // Post-barrier not needed when looping to self. java_lang_ref_Reference::set_next_raw(obj, obj); + oopDesc::bs()->write_ref_field(java_lang_ref_Reference::next_addr(obj), obj); if (next_d == obj) { // obj is last // Swap refs_list into pending_list_addr and // set obj's discovered to what we read from pending_list_addr. diff --git a/src/share/vm/memory/sharedHeap.cpp b/src/share/vm/memory/sharedHeap.cpp --- a/src/share/vm/memory/sharedHeap.cpp +++ b/src/share/vm/memory/sharedHeap.cpp @@ -37,21 +37,6 @@ SharedHeap* SharedHeap::_sh; -// The set of potentially parallel tasks in strong root scanning. -enum SH_process_strong_roots_tasks { - SH_PS_Universe_oops_do, - SH_PS_JNIHandles_oops_do, - SH_PS_ObjectSynchronizer_oops_do, - SH_PS_FlatProfiler_oops_do, - SH_PS_Management_oops_do, - SH_PS_SystemDictionary_oops_do, - SH_PS_ClassLoaderDataGraph_oops_do, - SH_PS_jvmti_oops_do, - SH_PS_CodeCache_oops_do, - // Leave this one last. - SH_PS_NumElements -}; - SharedHeap::SharedHeap(CollectorPolicy* policy_) : CollectedHeap(), _collector_policy(policy_), @@ -105,7 +90,7 @@ class AssertNonScavengableClosure: public OopClosure { public: virtual void do_oop(oop* p) { - assert(!Universe::heap()->is_in_partial_collection(*p), + assert(*p == NULL || !Universe::heap()->is_in_partial_collection(*p), "Referent should not be scavengable."); } virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } }; @@ -139,7 +124,8 @@ void SharedHeap::process_strong_roots(bool activate_scope, ScanningOption so, OopClosure* roots, - KlassClosure* klass_closure) { + KlassClosure* klass_closure, + GCPhaseTimeTracker* phase_durations) { StrongRootsScope srs(this, activate_scope); // General strong roots. @@ -148,12 +134,19 @@ // in a method not running in a GC worker. Otherwise the GC worker // could be trying to change the termination condition while the task // is executing in another GC worker. - if (!_process_strong_tasks->is_task_claimed(SH_PS_Universe_oops_do)) { - Universe::oops_do(roots); + { + TrackPhaseTime x(phase_durations, SH_PS_Universe_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_Universe_oops_do)) { + Universe::oops_do(roots); + } } - // Global (strong) JNI handles - if (!_process_strong_tasks->is_task_claimed(SH_PS_JNIHandles_oops_do)) - JNIHandles::oops_do(roots); + { + // Global (strong) JNI handles + TrackPhaseTime x(phase_durations, SH_PS_JNIHandles_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_JNIHandles_oops_do)) { + JNIHandles::oops_do(roots); + } + } CodeBlobToOopClosure code_roots(roots, true); @@ -161,70 +154,99 @@ // If we limit class scanning to SO_SystemClasses we need to apply a CLD closure to // CLDs which are strongly reachable from the thread stacks. CLDToOopClosure* roots_from_clds_p = ((so & SO_SystemClasses) ? &roots_from_clds : NULL); - // All threads execute this; the individual threads are task groups. - if (CollectedHeap::use_parallel_gc_threads()) { - Threads::possibly_parallel_oops_do(roots, roots_from_clds_p, &code_roots); - } else { - Threads::oops_do(roots, roots_from_clds_p, &code_roots); + { + // All threads execute this; the individual threads are task groups. + TrackPhaseTime x(phase_durations, SH_PS_Threads_oops_do); + if (CollectedHeap::use_parallel_gc_threads()) { + Threads::possibly_parallel_oops_do(roots, roots_from_clds_p, &code_roots); + } else { + Threads::oops_do(roots, roots_from_clds_p, &code_roots); + } } - - if (!_process_strong_tasks-> is_task_claimed(SH_PS_ObjectSynchronizer_oops_do)) - ObjectSynchronizer::oops_do(roots); - if (!_process_strong_tasks->is_task_claimed(SH_PS_FlatProfiler_oops_do)) - FlatProfiler::oops_do(roots); - if (!_process_strong_tasks->is_task_claimed(SH_PS_Management_oops_do)) - Management::oops_do(roots); - if (!_process_strong_tasks->is_task_claimed(SH_PS_jvmti_oops_do)) - JvmtiExport::oops_do(roots); - - if (!_process_strong_tasks->is_task_claimed(SH_PS_SystemDictionary_oops_do)) { - if (so & SO_AllClasses) { - SystemDictionary::oops_do(roots); - } else if (so & SO_SystemClasses) { - SystemDictionary::always_strong_oops_do(roots); - } else { - fatal("We should always have selected either SO_AllClasses or SO_SystemClasses"); + { + TrackPhaseTime x(phase_durations, SH_PS_ObjectSynchronizer_oops_do); + if (!_process_strong_tasks-> is_task_claimed(SH_PS_ObjectSynchronizer_oops_do)) { + ObjectSynchronizer::oops_do(roots); + } + } + { + TrackPhaseTime x(phase_durations, SH_PS_FlatProfiler_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_FlatProfiler_oops_do)) { + FlatProfiler::oops_do(roots); + } + } + { + TrackPhaseTime x(phase_durations, SH_PS_Management_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_Management_oops_do)) { + Management::oops_do(roots); + } + } + { + TrackPhaseTime x(phase_durations, SH_PS_jvmti_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_jvmti_oops_do)) { + JvmtiExport::oops_do(roots); + } + } + { + TrackPhaseTime x(phase_durations, SH_PS_SystemDictionary_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_SystemDictionary_oops_do)) { + if (so & SO_AllClasses) { + SystemDictionary::oops_do(roots); + } else if (so & SO_SystemClasses) { + SystemDictionary::always_strong_oops_do(roots); + } else { + fatal("We should always have selected either SO_AllClasses or SO_SystemClasses"); + } + } + } + + { + TrackPhaseTime x(phase_durations, SH_PS_ClassLoaderDataGraph_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_ClassLoaderDataGraph_oops_do)) { + if (so & SO_AllClasses) { + ClassLoaderDataGraph::oops_do(roots, klass_closure, /* must_claim */ false); + } else if (so & SO_SystemClasses) { + ClassLoaderDataGraph::always_strong_oops_do(roots, klass_closure, /* must_claim */ true); + } } } - if (!_process_strong_tasks->is_task_claimed(SH_PS_ClassLoaderDataGraph_oops_do)) { - if (so & SO_AllClasses) { - ClassLoaderDataGraph::oops_do(roots, klass_closure, /* must_claim */ false); - } else if (so & SO_SystemClasses) { - ClassLoaderDataGraph::always_strong_oops_do(roots, klass_closure, /* must_claim */ true); + { + // All threads execute the following. A specific chunk of buckets + // from the StringTable are the individual tasks. + TrackPhaseTime x(phase_durations, SH_PS_StringTable_oops_do); + if (so & SO_Strings) { + if (CollectedHeap::use_parallel_gc_threads()) { + StringTable::possibly_parallel_oops_do(roots); + } else { + StringTable::oops_do(roots); + } } } - // All threads execute the following. A specific chunk of buckets - // from the StringTable are the individual tasks. - if (so & SO_Strings) { - if (CollectedHeap::use_parallel_gc_threads()) { - StringTable::possibly_parallel_oops_do(roots); - } else { - StringTable::oops_do(roots); + { + TrackPhaseTime x(phase_durations, SH_PS_CodeCache_oops_do); + if (!_process_strong_tasks->is_task_claimed(SH_PS_CodeCache_oops_do)) { + if (so & SO_ScavengeCodeCache) { + assert(&code_roots != NULL, "must supply closure for code cache"); + + // We only visit parts of the CodeCache when scavenging. + CodeCache::scavenge_root_nmethods_do(&code_roots); + } + if (so & SO_AllCodeCache) { + assert(&code_roots != NULL, "must supply closure for code cache"); + + // CMSCollector uses this to do intermediate-strength collections. + // We scan the entire code cache, since CodeCache::do_unloading is not called. + CodeCache::blobs_do(&code_roots); + } + // Verify that the code cache contents are not subject to + // movement by a scavenging collection. + DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, /*do_marking=*/ false)); + DEBUG_ONLY(CodeCache::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable)); } } - if (!_process_strong_tasks->is_task_claimed(SH_PS_CodeCache_oops_do)) { - if (so & SO_ScavengeCodeCache) { - assert(&code_roots != NULL, "must supply closure for code cache"); - - // We only visit parts of the CodeCache when scavenging. - CodeCache::scavenge_root_nmethods_do(&code_roots); - } - if (so & SO_AllCodeCache) { - assert(&code_roots != NULL, "must supply closure for code cache"); - - // CMSCollector uses this to do intermediate-strength collections. - // We scan the entire code cache, since CodeCache::do_unloading is not called. - CodeCache::blobs_do(&code_roots); - } - // Verify that the code cache contents are not subject to - // movement by a scavenging collection. - DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, /*do_marking=*/ false)); - DEBUG_ONLY(CodeCache::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable)); - } - _process_strong_tasks->all_tasks_completed(); } diff --git a/src/share/vm/memory/sharedHeap.hpp b/src/share/vm/memory/sharedHeap.hpp --- a/src/share/vm/memory/sharedHeap.hpp +++ b/src/share/vm/memory/sharedHeap.hpp @@ -46,6 +46,7 @@ class FlexibleWorkGang; class CollectorPolicy; class KlassClosure; +class GCPhaseTimeTracker; // Note on use of FlexibleWorkGang's for GC. // There are three places where task completion is determined. @@ -107,7 +108,23 @@ friend class VM_GC_Operation; friend class VM_CGC_Operation; - +public: + // The set of potentially parallel tasks in strong root scanning. + enum SH_process_strong_roots_tasks { + SH_PS_Universe_oops_do, + SH_PS_JNIHandles_oops_do, + SH_PS_ObjectSynchronizer_oops_do, + SH_PS_FlatProfiler_oops_do, + SH_PS_Management_oops_do, + SH_PS_SystemDictionary_oops_do, + SH_PS_ClassLoaderDataGraph_oops_do, + SH_PS_jvmti_oops_do, + SH_PS_CodeCache_oops_do, + SH_PS_Threads_oops_do, + SH_PS_StringTable_oops_do, + // Leave this one last. + SH_PS_NumElements + }; private: // For claiming strong_roots tasks. SubTasksDone* _process_strong_tasks; @@ -238,7 +255,8 @@ void process_strong_roots(bool activate_scope, ScanningOption so, OopClosure* roots, - KlassClosure* klass_closure); + KlassClosure* klass_closure, + GCPhaseTimeTracker* phase_durations = NULL); // Apply "root_closure" to the JNI weak roots.. void process_weak_roots(OopClosure* root_closure); diff --git a/src/share/vm/memory/space.cpp b/src/share/vm/memory/space.cpp --- a/src/share/vm/memory/space.cpp +++ b/src/share/vm/memory/space.cpp @@ -269,8 +269,8 @@ bool mangle_space) { HeapWord* bottom = mr.start(); HeapWord* end = mr.end(); - assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), - "invalid space boundaries"); + //assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), + // "invalid space boundaries"); set_bottom(bottom); set_end(end); if (clear_space) clear(mangle_space); @@ -678,7 +678,7 @@ void ContiguousSpace::oop_iterate(ExtendedOopClosure* blk) { if (is_empty()) return; - HeapWord* obj_addr = bottom(); + HeapWord* obj_addr = first_object_start(); HeapWord* t = top(); // Could call objects iterate, but this is easier. while (obj_addr < t) { @@ -725,7 +725,7 @@ void ContiguousSpace::object_iterate(ObjectClosure* blk) { if (is_empty()) return; - WaterMark bm = bottom_mark(); + WaterMark bm = first_object_mark(); object_iterate_from(bm, blk); } diff --git a/src/share/vm/memory/space.hpp b/src/share/vm/memory/space.hpp --- a/src/share/vm/memory/space.hpp +++ b/src/share/vm/memory/space.hpp @@ -141,6 +141,8 @@ virtual void set_bottom(HeapWord* value) { _bottom = value; } virtual void set_end(HeapWord* value) { _end = value; } + virtual HeapWord* first_object_start() const { return bottom(); } + virtual HeapWord* saved_mark_word() const { return _saved_mark_word; } void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; } @@ -188,8 +190,8 @@ virtual void mangle_region(MemRegion mr) {} // Testers - bool is_empty() const { return used() == 0; } - bool not_empty() const { return used() > 0; } + virtual bool is_empty() const { return used() == 0; } + //bool not_empty() const { return used() > 0; } // Returns true iff the given the space contains the // given address as part of an allocated object. For @@ -417,9 +419,9 @@ // Used temporarily during a compaction phase to hold the value // top should have when compaction is complete. - HeapWord* compaction_top() const { return _compaction_top; } + virtual HeapWord* compaction_top() const { return _compaction_top; } - void set_compaction_top(HeapWord* value) { + virtual void set_compaction_top(HeapWord* value) { assert(value == NULL || (value >= bottom() && value <= end()), "should point inside space"); _compaction_top = value; @@ -805,15 +807,16 @@ virtual void clear(bool mangle_space); // Accessors - HeapWord* top() const { return _top; } - void set_top(HeapWord* value) { _top = value; } + virtual HeapWord* top() const { return _top; } + virtual void set_top(HeapWord* value) { _top = value; } virtual void set_saved_mark() { _saved_mark_word = top(); } void reset_saved_mark() { _saved_mark_word = bottom(); } - WaterMark bottom_mark() { return WaterMark(this, bottom()); } - WaterMark top_mark() { return WaterMark(this, top()); } - WaterMark saved_mark() { return WaterMark(this, saved_mark_word()); } + WaterMark bottom_mark() { return WaterMark(this, bottom()); } + WaterMark first_object_mark() { return WaterMark(this, first_object_start()); } + WaterMark top_mark() { return WaterMark(this, top()); } + WaterMark saved_mark() { return WaterMark(this, saved_mark_word()); } bool saved_mark_at_top() const { return saved_mark_word() == top(); } // In debug mode mangle (write it with a particular bit diff --git a/src/share/vm/runtime/timer.cpp b/src/share/vm/runtime/timer.cpp --- a/src/share/vm/runtime/timer.cpp +++ b/src/share/vm/runtime/timer.cpp @@ -26,6 +26,7 @@ #include "oops/oop.inline.hpp" #include "runtime/timer.hpp" #include "utilities/ostream.hpp" +#include "gc_implementation/g1/g1GCPhaseTimes.hpp" #ifdef TARGET_OS_FAMILY_linux # include "os_linux.inline.hpp" #endif @@ -155,6 +156,20 @@ } } +TrackPhaseTime::TrackPhaseTime(GCPhaseTimeTracker *data, uint phase) : + _data(data), _phase(phase) { + if (_data != NULL && _data->active()) { + _last = os::elapsed_counter(); + } +} + +TrackPhaseTime::~TrackPhaseTime() { + if (_data != NULL && _data->active()) { + double time = (double)(os::elapsed_counter() - _last) * 1000.0 / os::elapsed_frequency(); + _data->set_value(_phase, time); + } +} + TraceCPUTime::TraceCPUTime(bool doit, bool print_cr, outputStream *logfile) : diff --git a/src/share/vm/runtime/timer.hpp b/src/share/vm/runtime/timer.hpp --- a/src/share/vm/runtime/timer.hpp +++ b/src/share/vm/runtime/timer.hpp @@ -27,6 +27,8 @@ #include "utilities/globalDefinitions.hpp" +class GCPhaseTimeTracker; + // Timers for simple measurement. class elapsedTimer VALUE_OBJ_CLASS_SPEC { @@ -103,6 +105,17 @@ void resume() { if (_active) _t.start(); } }; +class TrackPhaseTime VALUE_OBJ_CLASS_SPEC { + private: + GCPhaseTimeTracker* _data; + uint _phase; + jlong _last; + public: + // Constructors + TrackPhaseTime(GCPhaseTimeTracker *data, uint phase); + ~TrackPhaseTime(); +}; + class TraceCPUTime: public StackObj { private: bool _active; // true if times will be measured and printed